From fcdac53fa4eda1640a0166157806d7c51ed62a41 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Wed, 25 Jun 2025 17:05:18 -0400 Subject: [PATCH 01/29] DOCS-4086: code samples for building a good dataset --- .github/workflows/update_sdk_methods.py | 2 +- docs/data-ai/_index.md | 6 +- docs/data-ai/ai/_index.md | 4 +- docs/data-ai/ai/act.md | 2 +- docs/data-ai/ai/advanced/_index.md | 8 - docs/data-ai/ai/create-dataset.md | 341 -------- docs/data-ai/ai/deploy.md | 2 +- docs/data-ai/capture-data/capture-sync.md | 2 +- docs/data-ai/data/_index.md | 2 +- docs/data-ai/train/_index.md | 11 + docs/data-ai/train/annotate-images.md | 459 ++++++++++ docs/data-ai/train/capture-images.md | 704 ++++++++++++++++ docs/data-ai/train/create-dataset.md | 157 ++++ docs/data-ai/{ai => train}/train-tflite.md | 17 +- docs/data-ai/{ai => train}/train.md | 5 +- docs/data-ai/train/update-dataset.md | 796 ++++++++++++++++++ .../upload-external-data.md | 17 +- docs/dev/_index.md | 2 +- docs/dev/reference/changelog.md | 12 +- docs/dev/reference/glossary/ml.md | 2 +- docs/dev/tools/cli.md | 6 +- .../teleoperate/default-interface.md | 2 +- .../get-started/supported-hardware/_index.md | 2 +- .../reference/advanced-modules/_index.md | 2 +- .../reference/services/vision/mlmodel.md | 4 +- docs/tutorials/projects/helmet.md | 4 +- .../tutorials/projects/send-security-photo.md | 2 +- .../tutorials/projects/verification-system.md | 2 +- layouts/docs/tutorials.html | 2 +- .../include/app/apis/generated/mltraining.md | 2 +- .../mltraining.SubmitCustomTrainingJob.md | 2 +- 31 files changed, 2180 insertions(+), 401 deletions(-) delete mode 100644 docs/data-ai/ai/advanced/_index.md delete mode 100644 docs/data-ai/ai/create-dataset.md create mode 100644 docs/data-ai/train/_index.md create mode 100644 docs/data-ai/train/annotate-images.md create mode 100644 docs/data-ai/train/capture-images.md create mode 100644 docs/data-ai/train/create-dataset.md rename docs/data-ai/{ai => train}/train-tflite.md (91%) rename docs/data-ai/{ai => train}/train.md (99%) create mode 100644 docs/data-ai/train/update-dataset.md rename docs/data-ai/{ai/advanced => train}/upload-external-data.md (97%) diff --git a/.github/workflows/update_sdk_methods.py b/.github/workflows/update_sdk_methods.py index 527516c7c4..7ba37e3a2f 100755 --- a/.github/workflows/update_sdk_methods.py +++ b/.github/workflows/update_sdk_methods.py @@ -342,7 +342,7 @@ "frame": "/services/frame-system/", "Viam app": "https://app.viam.com/", "organization settings page": "/manage/reference/organize/", - "image tags": "/data-ai/ai/create-dataset/#label-your-images", + "image tags": "/data-ai/train/create-dataset/#label-your-images", "API key": "/fleet/cli/#authenticate", "board model": "/dev/reference/apis/components/board/" } diff --git a/docs/data-ai/_index.md b/docs/data-ai/_index.md index 9554e6f352..6ac4dbd307 100644 --- a/docs/data-ai/_index.md +++ b/docs/data-ai/_index.md @@ -44,9 +44,9 @@ You can also monitor your machines through teleop, power your application logic, {{< how-to-expand "Leverage AI" "8" "INTERMEDIATE" "" "data-platform-ai" >}} {{< cards >}} -{{% card link="/data-ai/ai/create-dataset/" noimage="true" %}} -{{% card link="/data-ai/ai/train-tflite/" noimage="true" %}} -{{% card link="/data-ai/ai/train/" noimage="true" %}} +{{% card link="/data-ai/train/create-dataset/" noimage="true" %}} +{{% card link="/data-ai/train/train-tflite/" noimage="true" %}} +{{% card link="/data-ai/train/train/" noimage="true" %}} {{% card link="/data-ai/ai/deploy/" noimage="true" %}} {{% card link="/data-ai/ai/run-inference/" noimage="true" %}} {{% card link="/data-ai/ai/alert/" noimage="true" %}} diff --git a/docs/data-ai/ai/_index.md b/docs/data-ai/ai/_index.md index a7a53e72e5..e3a9a13527 100644 --- a/docs/data-ai/ai/_index.md +++ b/docs/data-ai/ai/_index.md @@ -1,6 +1,6 @@ --- -linkTitle: "Leverage AI" -title: "Leverage AI" +linkTitle: "Infer with ML models" +title: "Infer with ML models" weight: 300 layout: "empty" type: "docs" diff --git a/docs/data-ai/ai/act.md b/docs/data-ai/ai/act.md index 8fbd1ca309..b66b36a3a7 100644 --- a/docs/data-ai/ai/act.md +++ b/docs/data-ai/ai/act.md @@ -5,7 +5,7 @@ weight: 70 layout: "docs" type: "docs" description: "Use the vision service API to act based on inferences." -next: "/data-ai/ai/advanced/upload-external-data/" +next: "/data-ai/train/upload-external-data/" --- You can use the [vision service API](/dev/reference/apis/services/vision/) to get information about your machine's inferences and program behavior based on that. diff --git a/docs/data-ai/ai/advanced/_index.md b/docs/data-ai/ai/advanced/_index.md deleted file mode 100644 index f004c032cf..0000000000 --- a/docs/data-ai/ai/advanced/_index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -linkTitle: "Advanced" -title: "Advanced" -weight: 200 -layout: "empty" -type: "docs" -empty_node: true ---- diff --git a/docs/data-ai/ai/create-dataset.md b/docs/data-ai/ai/create-dataset.md deleted file mode 100644 index b08766a59f..0000000000 --- a/docs/data-ai/ai/create-dataset.md +++ /dev/null @@ -1,341 +0,0 @@ ---- -linkTitle: "Create a dataset" -title: "Create a dataset" -weight: 10 -layout: "docs" -type: "docs" -description: "Create a dataset to train a machine learning model." -aliases: - - /fleet/dataset/ - - /manage/data/label/ - - /manage/data/dataset/ - - /data/dataset/ ---- - -To ensure a machine learning model you create performs well, you need to train it on a variety of images that cover the range of things your machine should be able to recognize. - -To train a model, you need a dataset that meets the following criteria: - -- the dataset contains at least 15 images -- at least 80% of the images have labels -- for each selected label, at least 10 bounding boxes exist - -This page explains how to create a dataset that meets these criteria for your training purposes. - -## Prerequisites - -{{% expand "a machine connected to Viam" %}} - -{{% snippet "setup.md" %}} - -{{% /expand %}} - -{{% expand "a camera, connected to your machine, to capture images" %}} - -Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). - -{{% /expand%}} - -## Create a dataset - -To create a dataset, use the CLI or the web UI: - -{{< tabs >}} -{{% tab name="Web UI" %}} - -1. Open the [**DATASETS** tab on the **DATA** page](https://app.viam.com/data/datasets). - -1. Click the **+ Create dataset** button. - - {{< imgproc src="/services/data/create-dataset.png" alt="The **DATASET** tab of the **DATA** page, showing the **+ Create dataset** button." resize="800x" style="width:500px" class="imgzoom" >}} - -1. Enter a unique name for the dataset. - -1. Click the **Create dataset** button to create the dataset. - -{{% /tab %}} -{{% tab name="CLI" %}} - -1. First, install the Viam CLI and authenticate: - - {{< readfile "/static/include/how-to/install-cli.md" >}} - -1. [Log in to the CLI](/dev/tools/cli/#authenticate). - -1. Run the following command to create a dataset, replacing the `` and `` placeholders with your organization ID and a unique name for the dataset: - - ```sh {class="command-line" data-prompt="$"} - viam dataset create --org-id= --name= - ``` - -{{% /tab %}} -{{< /tabs >}} - -## Capture images - -{{< tabs >}} -{{% tab name="One image" %}} - -You can add images to a dataset directly from a camera or vision component feed in the machine's **CONTROL** or **CONFIGURATION** tabs. - -To add an image directly to a dataset from a visual feed, complete the following steps: - -1. Open the **TEST** panel of any camera or vision service component to view a feed of images from the camera. -1. Click the button marked with the camera icon to save the currently displayed image to a dataset: - {{< imgproc src="/components/camera/add_image_to_dataset_button.png" alt="A button marked with the outline of a camera, emphasized in red" resize="800x" style="width:500px" class="imgzoom" >}} -1. Select an existing dataset. -1. Click **Add** to add the image to the selected dataset. -1. When you see a success notification that reads "Saved image to dataset", you have successfully added the image to the dataset. - -To view images added to your dataset, go to the **DATA** page's [**DATASETS** tab](https://app.viam.com/data/datasets) and select your dataset. - -{{% /tab %}} -{{% tab name="Many images" %}} - -To capture a large number of images for training an ML model, [Capture and sync image data](/data-ai/capture-data/capture-sync/) using the data management service with your camera. - -Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. -We recommend you tag the images first and then use the CLI to [add the tagged images to a dataset](/data-ai/ai/create-dataset/#add-tagged-images-to-a-dataset). - -{{< alert title="Tip" color="tip" >}} - -Once you have enough images, consider disabling data capture to [avoid incurring fees](https://www.viam.com/product/pricing) for capturing large amounts of training data. - -{{< /alert >}} -{{% /tab %}} -{{< /tabs >}} - -Once you've captured enough images for training, you must annotate them to train a model. - -## Annotate images - -Use the interface on the [**DATA** page](https://app.viam.com/data/view) to annotate your images. -Always follow best practices when you label your images: - -More data means better models - -: Incorporate as much data as you practically can to improve your model's overall performance. - -Include counterexamples - -: Include images with and without the object you’re looking to classify. -This helps the model distinguish the target object from the background and reduces the chances of false positives by teaching the model what the object is _not_. - -Avoid class imbalance - -: Don't train excessively on one specific type or class, make sure each category has a roughly equal number of images. -For instance, if you're training a dog detector, include images of various dog breeds to avoid bias towards one breed. -An imbalanced dataset can lead the model to favor one class over others, reducing its overall accuracy. - -Match training images to intended use case - -: Use images that reflect the quality and conditions of your production environment. -For example, if you plan to use a low-quality camera in production, train with low-quality images. -Similarly, if your model will run all day, capture images in daylight, nighttime, dusk, and dawn conditions. - -Vary angles and distances - -: Include image examples from every angle and distance that you expect the model to handle. - -Viam enables you to annotate images for the following machine learning methods: - -{{< tabs >}} -{{% tab name="Classification" %}} - -Classification determines a descriptive tag or set of tags for an image. -For example, classification could help you identify: - -- whether an image of a food display appears `full`, `empty`, or `average` -- the quality of manufacturing output: `good` or `bad` -- what combination of toppings exists on a pizza: `pepperoni`, `sausage` and `pepper`, or `pineapple` and `ham` and `mushroom` - -Viam supports single and multiple label classification. -To create a training set for classification, annotate tags to describe your images. - -To tag an image: - -1. Click on an image, then click the **+** next to the **Tags** option. -1. Add one or more tags to your image. - - {{}} - -Repeat these steps for all images in the dataset. - -{{% /tab %}} -{{% tab name="Object detection" %}} - -Object detection identifies and determines the location of certain objects in an image. -For example, object detection could help you identify: - -- how many `pizza` objects appear on a counter -- the number of `bicycle` and `pedestrian` objects on a greenway -- which `plant` objects are popular with `deer` in your garden - -To create a training set for object detection, annotate bounding boxes to teach your model to identify objects that you want to detect in future images. - -To label an object with a bounding box: - -1. Click on an image, then click the **Annotate** button in right side menu. -1. Choose an existing label or create a new label. -1. Holding the command key (on macOS), or the control key (on Linux and Windows), click and drag on the image to create the bounding box: - - {{}} - -{{< alert title="Tip" color="tip" >}} - -Once created, you can move, resize, or delete the bounding box. -{{< /alert >}} - -Repeat these steps for all images in the dataset. - -{{% /tab %}} -{{< /tabs >}} - -## Add tagged images to a dataset - -{{< tabs >}} -{{% tab name="Web UI" %}} - -1. Open the [**DATA** page](https://app.viam.com/data/view). - -1. Navigate to the **ALL DATA** tab. - -1. Use the checkbox in the upper left of each image to select labeled images. - -1. Click the **Add to dataset** button, select a dataset, and click the **Add ... images** button to add the selected images to the dataset. - -{{% /tab %}} -{{% tab name="CLI" %}} - -Use the Viam CLI to filter images by label and add the filtered images to a dataset: - -1. First, [create a dataset](#create-a-dataset), if you haven't already. - -1. If you just created a dataset, use the dataset ID output by the creation command. - If your dataset already exists, run the following command to get a list of dataset names and corresponding IDs: - - ```sh {class="command-line" data-prompt="$"} - viam dataset list - ``` - -1. Run the following [command](/dev/tools/cli/#dataset) to add all images labeled with a subset of tags to the dataset, replacing the `` placeholder with the dataset ID output by the command in the previous step: - - ```sh {class="command-line" data-prompt="$"} - viam dataset data add filter --dataset-id= --tags=red_star,blue_square - ``` - -{{% /tab %}} -{{% tab name="Data Client API" %}} - -The following script adds all images captured from a certain machine to a new dataset. Complete the following steps to use the script: - -1. Copy and paste the following code into a file named add_images_from_machine_to_dataset.py on your machine. - - ```python {class="line-numbers linkable-line-numbers" data-line="9-13" } - import asyncio - from typing import List, Optional - - from viam.rpc.dial import DialOptions, Credentials - from viam.app.viam_client import ViamClient - from viam.utils import create_filter - - # Configuration constants – replace with your actual values - DATASET_NAME = "" # a unique, new name for the dataset you want to create - ORG_ID = "" # your organization ID, find in your organization settings - PART_ID = "" # id of machine that captured target images, find in machine config - API_KEY = "" # API key, find or create in your organization settings - API_KEY_ID = "" # API key ID, find or create in your organization settings - - # Adjust the maximum number of images to add to the dataset - MAX_MATCHES = 500 - - async def connect() -> ViamClient: - """Establish a connection to the Viam client using API credentials.""" - dial_options = DialOptions( - credentials=Credentials( - type="api-key", - payload=API_KEY, - ), - auth_entity=API_KEY_ID, - ) - return await ViamClient.create_from_dial_options(dial_options) - - - async def fetch_binary_data_ids(data_client, part_id: str) -> List[str]: - """Fetch binary data metadata and return a list of BinaryData objects.""" - data_filter = create_filter(part_id=part_id) - all_matches = [] - last: Optional[str] = None - - print("Getting data for part...") - - while len(all_matches) < MAX_MATCHES: - print("Fetching more data...") - data, _, last = await data_client.binary_data_by_filter( - data_filter, - limit=50, - last=last, - include_binary_data=False, - ) - if not data: - break - all_matches.extend(data) - - return all_matches - - - async def main() -> int: - """Main execution function.""" - viam_client = await connect() - data_client = viam_client.data_client - - matching_data = await fetch_binary_data_ids(data_client, PART_ID) - - print("Creating dataset...") - - try: - dataset_id = await data_client.create_dataset( - name=DATASET_NAME, - organization_id=ORG_ID, - ) - print(f"Created dataset: {dataset_id}") - except Exception as e: - print("Error creating dataset. It may already exist.") - print("See: https://app.viam.com/data/datasets") - print(f"Exception: {e}") - return 1 - - print("Adding data to dataset...") - - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[obj.metadata.binary_data_id for obj in matching_data], - dataset_id=dataset_id - ) - - print("Added files to dataset.") - print(f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}") - - viam_client.close() - return 0 - - - if __name__ == "__main__": - asyncio.run(main()) - ``` - -1. Fill in the placeholders with values for your own organization, API key, machine, and dataset. - -1. Install the [Viam Python SDK](https://python.viam.dev/) by running the following command: - - ```sh {class="command-line" data-prompt="$"} - pip install viam-sdk - ``` - -1. Finally, run the following command to add the images to the dataset: - - ```sh {class="command-line" data-prompt="$"} - python add_images_from_machine_to_dataset.py - ``` - -{{% /tab %}} -{{< /tabs >}} diff --git a/docs/data-ai/ai/deploy.md b/docs/data-ai/ai/deploy.md index 5a7dc28b0f..251949cb6a 100644 --- a/docs/data-ai/ai/deploy.md +++ b/docs/data-ai/ai/deploy.md @@ -75,7 +75,7 @@ Save your config to use your specified version of the ML model. The service works with models trained on Viam or elsewhere: -- You can [train TFlite](/data-ai/ai/train-tflite/) or [other model frameworks](/data-ai/ai/train/) on data from your machines. +- You can [train TFlite](/data-ai/train/train-tflite/) or [other model frameworks](/data-ai/train/train/) on data from your machines. - You can use [ML models](https://app.viam.com/registry?type=ML+Model) from the [registry](https://app.viam.com/registry). - You can upload externally trained models from a model file on the [**MODELS** tab](https://app.viam.com/models). - You can use a [model](/data-ai/ai/deploy/#deploy-your-ml-model-on-an-ml-model-service) trained outside the Viam platform whose files are on your machine. See the documentation of the model of ML model service you're using (pick one that supports your model framework) for instructions on this. diff --git a/docs/data-ai/capture-data/capture-sync.md b/docs/data-ai/capture-data/capture-sync.md index e49114f86b..68b782f233 100644 --- a/docs/data-ai/capture-data/capture-sync.md +++ b/docs/data-ai/capture-data/capture-sync.md @@ -139,4 +139,4 @@ For other ways to control data synchronization, see: ## Next steps For more information on available configuration attributes and options like capturing directly to MongoDB or conditional sync, see [Advanced data capture and sync configurations](/data-ai/capture-data/advanced/advanced-data-capture-sync/). -To leverage AI, you can now [create a dataset](/data-ai/ai/create-dataset/) with the data you've captured. +To leverage AI, you can now [create a dataset](/data-ai/train/create-dataset/) with the data you've captured. diff --git a/docs/data-ai/data/_index.md b/docs/data-ai/data/_index.md index 422b01922b..ed4ebdbf3e 100644 --- a/docs/data-ai/data/_index.md +++ b/docs/data-ai/data/_index.md @@ -1,7 +1,7 @@ --- linkTitle: "Work with data" title: "Work with data" -weight: 200 +weight: 150 layout: "empty" type: "docs" empty_node: true diff --git a/docs/data-ai/train/_index.md b/docs/data-ai/train/_index.md new file mode 100644 index 0000000000..d0376f093a --- /dev/null +++ b/docs/data-ai/train/_index.md @@ -0,0 +1,11 @@ +--- +linkTitle: "Train an ML model" +title: "Train an ML model" +weight: 200 +layout: "empty" +type: "docs" +empty_node: true +open_on_desktop: true +header_only: true +noedit: true +--- diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/annotate-images.md new file mode 100644 index 0000000000..f91b41b73e --- /dev/null +++ b/docs/data-ai/train/annotate-images.md @@ -0,0 +1,459 @@ +--- +linkTitle: "Annotate images for training" +title: "Annotate images for training" +weight: 40 +layout: "docs" +type: "docs" +description: "Annotate images to train a machine learning model." +--- + +Use the interface on the [**DATA** page](https://app.viam.com/data/view) to annotate your images. +When you label your dataset, include: + +- images with and _without_ the categories you’re looking to identify +- a roughly equal number of images for each category +- images from your production environment, including lighting and camera quality +- examples from every angle and distance that you expect the model to handle + +Viam enables you to annotate images for use with the following machine learning methods. + +## Prerequisites + +{{% expand "A machine connected to Viam" %}} + +{{% snippet "setup.md" %}} + +{{% /expand %}} + +{{% expand "A camera, connected to your machine, to capture images" %}} + +Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). + +{{% /expand %}} + +## Classify images with tags + +Classification determines a descriptive tag or set of tags for an image. +For example, classification could help you identify: + +- whether an image of a food display appears `full`, `empty`, or `average` +- the quality of manufacturing output: `good` or `bad` +- what combination of toppings exists on a pizza: `pepperoni`, `sausage` and `pepper`, or `pineapple` and `ham` and `mushroom` + +Viam supports single and multiple label classification. +To create a training set for classification, annotate tags to describe your images. + +{{< tabs >}} +{{% tab name="Web UI" %}} + +To tag an image: + +1. Click on an image, then click the **+** next to the **Tags** option. +1. Add one or more tags to your image. + + {{}} + +Repeat these steps for all images in the dataset. + +{{% /tab %}} +{{% tab name="SDK" %}} + +Use an ML model to generate tags for an image. +The following code shows how to add tags to an image in Viam: + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +detector = VisionClient.from_robot(machine, "") + +# Get the captured data for a camera +result = await detector.capture_all_from_camera( + "", + return_image=True, + return_detections=True, +) +image = result.image +detections = result.detections + +tags = ["tag1", "tag2"] + +my_filter = create_filter(component_name="camera-1", organization_ids=[""]) +binary_metadata, count, last = await data_client.binary_data_by_filter( + filter=my_filter, + limit=20, + include_binary_data=False +) + +my_ids = [] + +for obj in binary_metadata: + my_ids.append( + obj.metadata.binary_data_id + ) + +binary_data = await data_client.add_tags_to_binary_data_by_ids(tags, my_ids) +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +ctx := context.Background() + +viamClient, err := client.New(ctx, "", logger) +if err != nil { + log.Fatal(err) +} +defer viamClient.Close(ctx) + +myDetector, err := vision.FromRobot(viamClient, "") +if err != nil { + log.Fatal(err) +} + +dataClient := viamClient.DataClient() + +// Get the captured data for a camera +result, err := myDetector.CaptureAllFromCamera(ctx, "", &vision.CaptureAllFromCameraRequest{ + ReturnImage: true, + ReturnDetections: true, +}) +if err != nil { + log.Fatal(err) +} + +image := result.Image +detections := result.Detections + +tags := []string{"tag1", "tag2"} + +myFilter := &datamanager.Filter{ + ComponentName: "camera-1", + OrganizationIDs: []string{""}, +} + +binaryResult, err := dataClient.BinaryDataByFilter(ctx, &datamanager.BinaryDataByFilterRequest{ + Filter: myFilter, + Limit: 20, + IncludeBinaryData: false, +}) +if err != nil { + log.Fatal(err) +} + +var myIDs []string +for _, obj := range binaryResult.BinaryMetadata { + myIDs = append(myIDs, obj.Metadata.BinaryDataID) +} + +_, err = dataClient.AddTagsToBinaryDataByIDs(ctx, &datamanager.AddTagsRequest{ + Tags: tags, + BinaryDataIDs: myIDs, +}) +if err != nil { + log.Fatal(err) +} +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +const client = await createViamClient(); +const myDetector = new VisionClient(client, ""); +const dataClient = client.dataClient; + +// Get the captured data for a camera +const result = await myDetector.captureAllFromCamera( + "", + { + returnImage: true, + returnDetections: true, + } +); +const image = result.image; +const detections = result.detections; + +const tags = ["tag1", "tag2"]; + +const myFilter = createFilter({ + componentName: "camera-1", + organizationIds: [""] +}); + +const binaryResult = await dataClient.binaryDataByFilter({ + filter: myFilter, + limit: 20, + includeBinaryData: false +}); + +const myIds: string[] = []; + +for (const obj of binaryResult.binaryMetadata) { + myIds.push(obj.metadata.binaryDataId); +} + +await dataClient.addTagsToBinaryDataByIds(tags, myIds); +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart + final viamClient = await ViamClient.connect(); + final myDetector = VisionClient.fromRobot(viamClient, ""); + final dataClient = viamClient.dataClient; + + // Get the captured data for a camera + final result = await myDetector.captureAllFromCamera( + "", + returnImage: true, + returnDetections: true, + ); + final image = result.image; + final detections = result.detections; + + final tags = ["tag1", "tag2"]; + + final myFilter = createFilter( + componentName: "camera-1", + organizationIds: [""], + ); + + final binaryResult = await dataClient.binaryDataByFilter( + filter: myFilter, + limit: 20, + includeBinaryData: false, + ); + + final myIds = []; + + for (final obj in binaryResult.binaryMetadata) { + myIds.add(obj.metadata.binaryDataId); + } + + await dataClient.addTagsToBinaryDataByIds(tags, myIds); +``` + +{{% /tab %}} +{{< /tabs >}} + +{{% /tab %}} +{{< /tabs >}} + +Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. + +## Detect objects with bounding boxes + +Object detection identifies and determines the location of certain objects in an image. +For example, object detection could help you identify: + +- how many `pizza` objects appear on a counter +- the number of `bicycle` and `pedestrian` objects on a greenway +- which `plant` objects are popular with `deer` in your garden + +To create a training set for object detection, annotate bounding boxes to teach your model to identify objects that you want to detect in future images. + +{{< tabs >}} +{{% tab name="Web UI" %}} + +To label an object with a bounding box: + +1. Click on an image, then click the **Annotate** button in right side menu. +1. Choose an existing label or create a new label. +1. Holding the command key (on macOS), or the control key (on Linux and Windows), click and drag on the image to create the bounding box: + + {{}} + +{{< alert title="Tip" color="tip" >}} + +Once created, you can move, resize, or delete the bounding box. +{{< /alert >}} + +Repeat these steps for all images in the dataset. + +{{% /tab %}} +{{% tab name="SDK" %}} + +Use an ML model to generate bounding boxes for an image. +The following code shows how to add bounding boxes to an image in Viam: + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +detector = VisionClient.from_robot(machine, "") + +# Get the captured data for a camera + +# Initialize data client +data_client = DataClient.create_from_dial_options( + dial_options=DialOptions( + credentials=Credentials( + type="api-key", + payload="" + ) + ), + organization_id="", + location_id="" +) + +# Initialize vision service +detector = VisionClient.from_robot(machine, "") + +# Capture data with image and detections +result = await detector.capture_all_from_camera( + "", + return_image=True, + return_detections=True, +) + +image = result.image +detections = result.detections + +# Upload image to obtain binary ID +binary_id = await data_client.binary_data_capture_upload( + binary_data=image.data, + part_id="", + component_type="camera", + component_name="", + method_name="get_image", + file_extension=".jpg" +) + +# Process each detection and create bounding boxes +for detection in detections: + bbox = detection.bounding_box + + # Create bounding box annotation + bbox_id = await data_client.add_bounding_box_to_image_by_id( + binary_id=binary_id.file_id, + label=detection.class_name, + x_min_normalized=bbox.x_min_normalized, + y_min_normalized=bbox.y_min_normalized, + x_max_normalized=bbox.x_max_normalized, + y_max_normalized=bbox.y_max_normalized + ) + + print(f"Bounding box created: {bbox_id} for class: {detection.class_name}") +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +ctx := context.Background() + +viamClient, err := client.New(ctx, "", logger) +if err != nil { + log.Fatal(err) +} +defer viamClient.Close(ctx) + +myDetector, err := vision.FromRobot(viamClient, "") +if err != nil { + log.Fatal(err) +} + +dataClient := viamClient.DataClient() + +// Get the captured data for a camera +result, err := myDetector.CaptureAllFromCamera(ctx, "", &vision.CaptureAllFromCameraRequest{ + ReturnImage: true, + ReturnDetections: true, +}) +if err != nil { + log.Fatal(err) +} + +image := result.Image +detections := result.Detections + +for _, detection := range detections { + bboxID, err := dataClient.AddBoundingBoxToImageByID(ctx, &datamanager.AddBoundingBoxRequest{ + BinaryID: "", + Label: detection.Label, + XMinNormalized: detection.BoundingBox.Min.X, + YMinNormalized: detection.BoundingBox.Min.Y, + XMaxNormalized: detection.BoundingBox.Max.X, + YMaxNormalized: detection.BoundingBox.Max.Y, + }) + + fmt.Printf("Added bounding box ID: %s for detection: %s\n", bboxID, detection.ClassName) +} +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +const client = await createViamClient(); +const myDetector = new VisionClient(client, ""); +const dataClient = client.dataClient; + +// Get the captured data for a camera +const result = await myDetector.captureAllFromCamera( + "", + { + returnImage: true, + returnDetections: true, + } +); +const image = result.image; +const detections = result.detections; + +// Process each detection and add bounding boxes +for (const detection of detections) { + const bboxId = await dataClient.addBoundingBoxToImageById({ + binaryId: "", + label: detection.className, + xMinNormalized: detection.boundingBox.xMin, + yMinNormalized: detection.boundingBox.yMin, + xMaxNormalized: detection.boundingBox.xMax, + yMaxNormalized: detection.boundingBox.yMax + }); + + console.log(`Added bounding box ID: ${bboxId} for detection: ${detection.className}`); +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +final viamClient = await ViamClient.connect(); +final myDetector = VisionClient.fromRobot(viamClient, ""); +final dataClient = viamClient.dataClient; + +// Get the captured data for a camera +final result = await myDetector.captureAllFromCamera( + "", + returnImage: true, + returnDetections: true, +); +final image = result.image; +final detections = result.detections; + +// Process each detection and add bounding boxes +for (final detection in detections) { + final bboxId = await dataClient.addBoundingBoxToImageById( + binaryId: "", + label: detection.className, + xMinNormalized: detection.boundingBox.xMin, + yMinNormalized: detection.boundingBox.yMin, + xMaxNormalized: detection.boundingBox.xMax, + yMaxNormalized: detection.boundingBox.yMax, + ); + + print('Added bounding box ID: $bboxId for detection: ${detection.className}'); +} +``` + +{{% /tab %}} +{{< /tabs >}} + +{{% /tab %}} +{{< /tabs >}} + +Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. \ No newline at end of file diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md new file mode 100644 index 0000000000..1a2c4d76e4 --- /dev/null +++ b/docs/data-ai/train/capture-images.md @@ -0,0 +1,704 @@ +--- +linkTitle: "Capture images for training" +title: "Capture images for training" +weight: 20 +layout: "docs" +type: "docs" +description: "Capture images to train a machine learning model." +--- + +## Prerequisites + +{{% expand "A machine connected to Viam" %}} + +{{% snippet "setup.md" %}} + +{{% /expand %}} + +{{% expand "A camera, connected to your machine, to capture images" %}} + +Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). + +{{% /expand%}} + +## Capture individual images + +{{< tabs >}} +{{% tab name="Web UI" %}} + +You can add images to a dataset directly from a camera or vision component feed in the machine's **CONTROL** or **CONFIGURATION** tabs. + +To add an image directly to a dataset from a visual feed, complete the following steps: + +1. Open the **TEST** panel of any camera or vision service component to view a feed of images from the camera. +1. Click the button marked with the camera icon to save the currently displayed image to a dataset: + {{< imgproc src="/components/camera/add_image_to_dataset_button.png" alt="A button marked with the outline of a camera, emphasized in red" resize="800x" style="width:500px" class="imgzoom" >}} +1. Select an existing dataset. +1. Click **Add** to add the image to the selected dataset. +1. When you see a success notification that reads "Saved image to dataset", you have successfully added the image to the dataset. + +To view images added to your dataset, go to the **DATA** page, open the [**DATASETS** tab](https://app.viam.com/data/datasets), then select your dataset. + +{{% /tab %}} +{{% tab name="SDK" %}} + +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Each SDK provides a method to add an image to a dataset using those IDs: + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +# Connect to Viam client +dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, +) + +viam_client = await ViamClient.create_from_dial_options(dial_options) +data_client = viam_client.data_client + +# Add image to dataset +await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[EXISTING_IMAGE_ID], + dataset_id=EXISTING_DATASET_ID +) + +print("Image added to dataset successfully") +viam_client.close() +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +// Connect to Viam client +client, err := app.NewViamClient( + context.Background() +) +if err != nil { + return fmt.Errorf("failed to create client: %w", err) +} +defer client.Close() + +// Authenticate +err = client.LoginWithAPIKey(context.Background(), APIKeyID, APIKey) +if err != nil { + return fmt.Errorf("failed to authenticate: %w", err) +} + +// Add image to dataset +err = client.AddBinaryDataToDatasetByIDs( + context.Background(), + []string{ExistingImageID}, + ExistingDatasetID, +) +if err != nil { + return fmt.Errorf("failed to add image to dataset: %w", err) +} + +fmt.Println("Image added to dataset successfully") +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +// Connect to Viam client +const client: ViamClient = await createViamClient({ + credential: { + type: 'api-key', + payload: API_KEY, + }, + authEntity: API_KEY_ID, +}); + +const dataClient = client.dataClient; + +// Add image to dataset +await dataClient.addBinaryDataToDatasetByIds({ + binaryIds: [EXISTING_IMAGE_ID], + datasetId: EXISTING_DATASET_ID, +}); + +console.log('Image added to dataset successfully'); +client.disconnect(); +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart + // Connect to Viam client + final client = await ViamClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + + final dataClient = client.dataClient; + + try { + // Add image to dataset + await dataClient.addBinaryDataToDatasetByIds( + binaryIds: [existingImageId], + datasetId: existingDatasetId, + ); + + print('Image added to dataset successfully'); + } finally { + await client.close(); + } +``` + +{{% /tab %}} +{{< /tabs >}} + +{{% /tab %}} +{{< /tabs >}} + +Once you've captured enough images for training, you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. + +## Capture images over time + +To capture a large number of images for training an ML model, [Capture and sync image data](/data-ai/capture-data/capture-sync/) using the data management service with your camera. + +Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. +We recommend you tag the images first and then use the CLI to [add the tagged images to a dataset](/data-ai/train/create-dataset/#add-tagged-images-to-a-dataset). + +{{< alert title="Tip" color="tip" >}} + +Once you have enough images, consider disabling data capture to [avoid incurring fees](https://www.viam.com/product/pricing) for capturing large amounts of training data. + +{{< /alert >}} + +Once you've captured enough images for training, you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. + +## Capture, annotate, and add images to a dataset + +The following example demonstrates how you can capture an image, use an ML model to generate annotations, then add the image to a dataset. You can use this logic to expand and improve your datasets continuously over time as your application runs. +Re-train your ML model on the improved dataset to improve the ML model. + +{{< tabs >}} +{{% tab name="Python" %}} +```python +import asyncio +import os +import time +from typing import Optional +from io import BytesIO + +from PIL import Image +from viam.app.app_client import AppClient +from viam.logging import getLogger + +# Global machine configuration +MACHINE_PART_ID = "your-machine-part-id-here" + +class DataCollector: + def __init__(self, component_name: str, dataset_id: str, api_key_id: str, api_key: str): + self.logger = getLogger(__name__) + self.component_name = component_name + self.dataset_id = dataset_id + self.api_key_id = api_key_id + self.api_key = api_key + + async def capture_and_store_image(self, processed_image: Image.Image, classification: str) -> None: + if not MACHINE_PART_ID: + raise ValueError("machine part ID not configured") + + # Create fresh data client connection + async with await self._create_data_client() as data_client: + image_data = self._encode_image_to_png(processed_image) + + # Generate unique filename with timestamp + timestamp = int(time.time()) + filename = f"{classification}-sample-{timestamp}.png" + + component_type = "rdk:component:camera" + + upload_metadata = { + "component_type": component_type, + "component_name": self.component_name, + "file_name": filename, + "file_extension": "png" + } + + try: + file_id = await data_client.file_upload_from_bytes( + part_id=MACHINE_PART_ID, + data=image_data, + **upload_metadata + ) + + # Associate file with dataset immediately + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[file_id], + dataset_id=self.dataset_id + ) + + self.logger.info( + f"successfully added {classification} image to dataset {self.dataset_id} " + f"(file ID: {file_id}, machine: {MACHINE_PART_ID})" + ) + + except Exception as e: + self.logger.error(f"failed to upload and associate image: {e}") + raise + + async def _create_data_client(self) -> AppClient: + if not self.api_key_id or not self.api_key: + raise ValueError("API credentials not configured") + + try: + client = AppClient.create_from_dial_options( + dial_options={ + "auth_entity": self.api_key_id, + "credentials": { + "type": "api-key", + "payload": self.api_key + } + } + ) + return client + + except Exception as e: + raise ValueError(f"failed to create app client: {e}") + + def _encode_image_to_png(self, img: Image.Image) -> bytes: + buffer = BytesIO() + img.save(buffer, format='PNG', optimize=True) + return buffer.getvalue() + +def create_data_collector(component_name: str, dataset_id: str, api_key_id: str, api_key: str) -> DataCollector: + if not component_name: + raise ValueError("component name is required") + if not dataset_id: + raise ValueError("dataset ID is required") + if not api_key_id or not api_key: + raise ValueError("API credentials are required") + + return DataCollector( + component_name=component_name, + dataset_id=dataset_id, + api_key_id=api_key_id, + api_key=api_key + ) + +# Example usage +async def main(): + collector = create_data_collector( + component_name="main_camera", + dataset_id="your-dataset-id", + api_key_id="your-api-key-id", + api_key="your-api-key" + ) + + # Example with PIL Image + sample_image = Image.new('RGB', (640, 480), color='red') + await collector.capture_and_store_image(sample_image, "positive_sample") + +if __name__ == "__main__": + asyncio.run(main()) +``` +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package datasetcreation + +import ( + "context" + "fmt" + "image" + "time" + + "go.viam.com/rdk/logging" + "go.viam.com/rdk/services/datamanager/app" + "go.viam.com/rdk/app/appclient" +) + +var MachinePartID string = "your-machine-part-id-here" + +type DataCollector struct { + logger logging.Logger + componentName string + datasetID string + apiKeyID string + apiKey string +} + +func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { + if MachinePartID == "" { + return fmt.Errorf("machine part ID not configured") + } + + // Create fresh data client connection + dataClient, err := dc.createDataClient(ctx) + if err != nil { + dc.logger.Errorf("failed to create data client: %v", err) + return fmt.Errorf("data client initialization failed: %w", err) + } + defer dataClient.Close() + + imageData, err := dc.encodeImageToPNG(processedImage) + if err != nil { + dc.logger.Errorf("image encoding failed: %v", err) + return fmt.Errorf("failed to encode image: %w", err) + } + + // Generate unique filename with timestamp + timestamp := time.Now().Unix() + filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) + + componentType := "rdk:component:camera" + fileExtension := "png" + + uploadOptions := app.FileUploadOptions{ + ComponentType: &componentType, + ComponentName: &dc.componentName, + FileName: &filename, + FileExtension: &fileExtension, + } + + fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) + if err != nil { + dc.logger.Errorf("file upload failed for %s: %v", filename, err) + return fmt.Errorf("failed to upload image: %w", err) + } + + // Associate file with dataset immediately + err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) + if err != nil { + dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) + return fmt.Errorf("failed to add image to dataset: %w", err) + } + + dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", + classification, dc.datasetID, fileID, MachinePartID) + + return nil +} + +func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { + if dc.apiKeyID == "" || dc.apiKey == "" { + return nil, fmt.Errorf("API credentials not configured") + } + + client, err := appclient.New(ctx, appclient.Config{ + KeyID: dc.apiKeyID, + Key: dc.apiKey, + }) + if err != nil { + return nil, fmt.Errorf("failed to create app client: %w", err) + } + + return client.DataClient, nil +} + +func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { + // PNG encoding implementation + return nil, nil // Placeholder +} + +func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { + if logger == nil { + return nil, fmt.Errorf("logger is required") + } + if componentName == "" { + return nil, fmt.Errorf("component name is required") + } + if datasetID == "" { + return nil, fmt.Errorf("dataset ID is required") + } + if apiKeyID == "" || apiKey == "" { + return nil, fmt.Errorf("API credentials are required") + } + + return &DataCollector{ + logger: logger, + componentName: componentName, + datasetID: datasetID, + apiKeyID: apiKeyID, + apiKey: apiKey, + }, nil +} +``` +{{% /tab %}} +{{% tab name="Typescript" %}} + +```typescript +import { createRobotClient, RobotClient } from '@viamrobotics/sdk'; +import { AppClient, DataClient } from '@viamrobotics/app-client'; +import { Logger } from '@viamrobotics/utils'; + +const MACHINE_PART_ID: string = 'your-machine-part-id-here'; + +interface FileUploadOptions { + componentType?: string; + componentName?: string; + fileName?: string; + fileExtension?: string; +} + +export class DataCollector { + private logger: Logger; + private componentName: string; + private datasetId: string; + private apiKeyId: string; + private apiKey: string; + + constructor( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string + ) { + this.logger = logger; + this.componentName = componentName; + this.datasetId = datasetId; + this.apiKeyId = apiKeyId; + this.apiKey = apiKey; + } + + async captureAndStoreImage( + processedImage: ArrayBuffer, + classification: string + ): Promise { + if (!MACHINE_PART_ID) { + throw new Error('Machine part ID not configured'); + } + + let dataClient: DataClient | null = null; + + try { + dataClient = await this.createDataClient(); + + const imageData = await this.encodeImageToPng(processedImage); + const timestamp = Math.floor(Date.now() / 1000); + const filename = `${classification}-sample-${timestamp}.png`; + + const componentType = 'rdk:component:camera'; + const fileExtension = 'png'; + + const uploadOptions: FileUploadOptions = { + componentType, + componentName: this.componentName, + fileName: filename, + fileExtension, + }; + + const fileId = await dataClient.fileUploadFromBytes( + MACHINE_PART_ID, + new Uint8Array(imageData), + uploadOptions + ); + + await dataClient.addBinaryDataToDatasetByIds( + [fileId], + this.datasetId + ); + + this.logger.info( + `Successfully added ${classification} image to dataset ${this.datasetId} ` + + `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})` + ); + + } catch (error) { + this.logger.error(`File upload failed for ${classification}: ${error}`); + throw new Error(`Failed to upload image: ${error}`); + } finally { + if (dataClient) { + await dataClient.close(); + } + } + } + + private async createDataClient(): Promise { + if (!this.apiKeyId || !this.apiKey) { + throw new Error('API credentials not configured'); + } + + const appClient = new AppClient({ + apiKeyId: this.apiKeyId, + apiKey: this.apiKey, + }); + + return appClient.dataClient(); + } + + private async encodeImageToPng(image: ArrayBuffer): Promise { + try { + // PNG encoding implementation would depend on your image processing library + // This is a placeholder - you would use a library like 'pngjs' or 'canvas' + return image; // Assuming image is already PNG encoded + } catch (error) { + this.logger.error(`Image encoding failed: ${error}`); + throw new Error(`Failed to encode image: ${error}`); + } + } + + static create( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string + ): DataCollector { + if (!logger) { + throw new Error('Logger is required'); + } + if (!componentName) { + throw new Error('Component name is required'); + } + if (!datasetId) { + throw new Error('Dataset ID is required'); + } + if (!apiKeyId || !apiKey) { + throw new Error('API credentials are required'); + } + + return new DataCollector( + logger, + componentName, + datasetId, + apiKeyId, + apiKey + ); + } +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'dart:typed_data'; +import 'dart:io'; +import 'package:viam_sdk/viam_sdk.dart'; +import 'package:viam_sdk/src/app/app_client.dart'; +import 'package:viam_sdk/src/app/data_client.dart'; +import 'package:image/image.dart' as img; + +const String machinePartId = 'your-machine-part-id-here'; + +class DataCollector { + final Logging logger; + final String componentName; + final String datasetId; + final String apiKeyId; + final String apiKey; + + DataCollector({ + required this.logger, + required this.componentName, + required this.datasetId, + required this.apiKeyId, + required this.apiKey, + }); + + Future captureAndStoreImage( + Image processedImage, + String classification, + ) async { + if (machinePartId.isEmpty) { + throw Exception('Machine part ID not configured'); + } + + DataClient? dataClient; + try { + dataClient = await _createDataClient(); + + final imageData = await _encodeImageToPng(processedImage); + final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; + final filename = '$classification-sample-$timestamp.png'; + + const componentType = 'rdk:component:camera'; + const fileExtension = 'png'; + + final uploadOptions = FileUploadOptions( + componentType: componentType, + componentName: componentName, + fileName: filename, + fileExtension: fileExtension, + ); + + final fileId = await dataClient.fileUploadFromBytes( + machinePartId, + imageData, + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds( + [fileId], + datasetId, + ); + + logger.info( + 'Successfully added $classification image to dataset $datasetId ' + '(file ID: $fileId, machine: $machinePartId)', + ); + } catch (error) { + logger.error('File upload failed for $classification: $error'); + rethrow; + } finally { + await dataClient?.close(); + } + } + + Future _createDataClient() async { + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw Exception('API credentials not configured'); + } + + final appClient = await AppClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + + return appClient.dataClient; + } + + Future _encodeImageToPng(Image image) async { + try { + final pngBytes = img.encodePng(image); + return Uint8List.fromList(pngBytes); + } catch (error) { + logger.error('Image encoding failed: $error'); + throw Exception('Failed to encode image: $error'); + } + } + + static DataCollector create({ + required Logging logger, + required String componentName, + required String datasetId, + required String apiKeyId, + required String apiKey, + }) { + if (componentName.isEmpty) { + throw ArgumentError('Component name is required'); + } + if (datasetId.isEmpty) { + throw ArgumentError('Dataset ID is required'); + } + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw ArgumentError('API credentials are required'); + } + + return DataCollector( + logger: logger, + componentName: componentName, + datasetId: datasetId, + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + } +} +``` + +{{% /tab %}} +{{< /tabs >}} diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md new file mode 100644 index 0000000000..e54c428bd8 --- /dev/null +++ b/docs/data-ai/train/create-dataset.md @@ -0,0 +1,157 @@ +--- +linkTitle: "Create training dataset" +title: "Create training dataset" +weight: 10 +layout: "docs" +type: "docs" +description: "Create a dataset to train a machine learning model." +aliases: + - /fleet/dataset/ + - /manage/data/label/ + - /manage/data/dataset/ + - /data/dataset/ + - /data-ai/ai/create-dataset/ +--- + +To create a dataset: + +{{< tabs >}} +{{% tab name="Web UI" %}} + +1. Navigate to the **DATA** page and open the [**DATASETS** tab](https://app.viam.com/data/datasets). + +1. Click the **+ Create dataset** button. + + {{< imgproc src="/services/data/create-dataset.png" alt="The **DATASET** tab of the **DATA** page, showing the **+ Create dataset** button." resize="800x" style="width:500px" class="imgzoom" >}} + +1. Enter a unique name for the dataset. + +1. Click **Create dataset** to create the dataset. + +{{% /tab %}} +{{% tab name="CLI" %}} + +1. First, install the Viam CLI and authenticate: + + {{< readfile "/static/include/how-to/install-cli.md" >}} + +1. [Log in to the CLI](/dev/tools/cli/#authenticate). + +1. Run the following command to create a dataset, replacing the `` and `` placeholders with your organization ID and a unique name for the dataset: + + ```sh {class="command-line" data-prompt="$"} + viam dataset create --org-id= --name= + ``` + +{{% /tab %}} +{{% tab name="SDK" %}} + +{{< tabs >}} +{{% tab name="Python" %}} + +To create a dataset, pass a unique dataset name and organization ID to `data_client.create_dataset`: + +```python +viam_client = await connect() +data_client = viam_client.data_client + +print("Creating dataset...") + +try: + dataset_id = await data_client.create_dataset( + name="", + organization_id="", + ) + print(f"Created dataset: {dataset_id}") +except Exception as e: + print("Error creating dataset. It may already exist.") + print(f"Exception: {e}") + return 1 +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +To create a dataset, pass a unique dataset name and organization ID to `dataClient.CreateDataset`: + +```go +ctx := context.Background() +viamClient, err := client.New(ctx, "", logger) +if err != nil { + log.Fatal(err) +} +defer viamClient.Close(ctx) + +dataClient := viamClient.DataClient() + +fmt.Println("Creating dataset...") + +datasetID, err := dataClient.CreateDataset(ctx, &datamanager.CreateDatasetRequest{ + Name: "", + OrganizationID: "", +}) + +if err != nil { + fmt.Println("Error creating dataset. It may already exist.") + fmt.Printf("Exception: %v\n", err) + return +} + +fmt.Printf("Created dataset: %s\n", datasetID) +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +To create a dataset, pass a unique dataset name and organization ID to `dataClient.createDataset`: + +```typescript +const client = await createViamClient(); +const dataClient = client.dataClient; + +console.log("Creating dataset..."); + +try { + const datasetId = await dataClient.createDataset({ + name: "", + organizationId: "", + }); + console.log(`Created dataset: ${datasetId}`); +} catch (error) { + console.log("Error creating dataset. It may already exist."); + console.log(`Exception: ${error}`); + process.exit(1); +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +To create a dataset, pass a unique dataset name and organization ID to `dataClient.createDataset`: + +```dart +final viamClient = await ViamClient.connect(); +final dataClient = viamClient.dataClient; + +print("Creating dataset..."); + +try { +final datasetId = await dataClient.createDataset( + name: "", + organizationId: "", +); + print("Created dataset: $datasetId"); +} catch (e) { + print("Error creating dataset. It may already exist."); + print("Exception: $e"); + return; +} +``` + +{{% /tab %}} +{{< /tabs >}} + +{{% /tab %}} +{{< /tabs >}} + +Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. \ No newline at end of file diff --git a/docs/data-ai/ai/train-tflite.md b/docs/data-ai/train/train-tflite.md similarity index 91% rename from docs/data-ai/ai/train-tflite.md rename to docs/data-ai/train/train-tflite.md index e51f87652b..7deecfca13 100644 --- a/docs/data-ai/ai/train-tflite.md +++ b/docs/data-ai/train/train-tflite.md @@ -1,7 +1,7 @@ --- linkTitle: "Train TFlite model" title: "Train a TFlite model" -weight: 20 +weight: 50 type: "docs" tags: ["vision", "data", "services"] images: ["/services/ml/train.svg"] @@ -19,6 +19,7 @@ aliases: - /extend/modular-resources/examples/tflite-module/ - /modular-resources/examples/tflite-module/ - /registry/examples/tflite-module/ + - /data-ai/ai/train-tflite/ languages: [] viamresources: ["data_manager", "mlmodel", "vision"] platformarea: ["ml"] @@ -37,9 +38,15 @@ Follow this guide to use your image data to train an ML model, so that your mach {{% /expand %}} -{{% expand "a dataset with labels" %}} +{{% expand "a dataset that meets training requirements" %}} -Follow the guide to [create a dataset](/data-ai/ai/create-dataset/). +To train a model, your dataset must meet the following criteria: + +- at least 15 images +- at least 80% of the images have labels +- for each training label, at least 10 bounding boxes + +Follow the guide to [create a dataset](/data-ai/train/create-dataset/). {{% /expand%}} @@ -145,11 +152,11 @@ Using this approach, each subsequent model version becomes more accurate than th To capture images of edge cases and re-train your model using those images, complete the following steps: -1. Add edge case images to your training dataset. You can find edge cases in your existing data on the [**DATA** page](https://app.viam.com/data/) or [capture new images and add them to your training dataset](/data-ai/ai/create-dataset/#capture-images). +1. Add edge case images to your training dataset. You can find edge cases in your existing data on the [**DATA** page](https://app.viam.com/data/) or [capture new images and add them to your training dataset](/data-ai/train/create-dataset/#capture-images). 1. Visit the **DATASET** tab of the **DATA** page and annotate the image. -1. Repeat the [steps above](/data-ai/ai/train-tflite/#train-a-machine-learning-model) to train and release a new version of your ML model. Your machines will automatically update to the new version of the model soon after release. +1. Repeat the [steps above](/data-ai/train/train-tflite/#train-a-machine-learning-model) to train and release a new version of your ML model. Your machines will automatically update to the new version of the model soon after release. ## Next steps diff --git a/docs/data-ai/ai/train.md b/docs/data-ai/train/train.md similarity index 99% rename from docs/data-ai/ai/train.md rename to docs/data-ai/train/train.md index 71caae8cda..610ae073dd 100644 --- a/docs/data-ai/ai/train.md +++ b/docs/data-ai/train/train.md @@ -2,7 +2,7 @@ linkTitle: "Train other models" title: "Train other models" tags: ["data management", "ml", "model training"] -weight: 30 +weight: 51 layout: "docs" type: "docs" aliases: @@ -10,6 +10,7 @@ aliases: - /how-tos/create-custom-training-scripts/ - /services/ml/training-scripts/ - /registry/training-scripts/ + - /data-ai/ai/train/ languages: ["python"] viamresources: ["mlmodel", "data_manager"] platformarea: ["ml"] @@ -27,7 +28,7 @@ If you wish to do this, skip to [Submit a training job](#submit-a-training-job). {{% expand "A dataset with data you can train an ML model on. Click to see instructions." %}} -For images, follow the instructions to [Create a dataset](/data-ai/ai/create-dataset/) to create a dataset and label data. +For images, follow the instructions to [Create a dataset](/data-ai/train/create-dataset/) to create a dataset and label data. For other data, use the [Data Client API](/dev/reference/apis/data-client/) from within the training script to store data in the Viam Cloud. diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md new file mode 100644 index 0000000000..15909e50ce --- /dev/null +++ b/docs/data-ai/train/update-dataset.md @@ -0,0 +1,796 @@ +--- +linkTitle: "Add to a training dataset" +title: "Add to a training dataset" +weight: 30 +layout: "docs" +type: "docs" +description: "Update a dataset used to train a machine learning model." +--- + +## Prerequisites + +{{% expand "A machine connected to Viam" %}} + +{{% snippet "setup.md" %}} + +{{% /expand %}} + +{{% expand "A camera, connected to your machine, to capture images" %}} + +Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). + +{{% /expand%}} + +## Add to a dataset + +{{< tabs >}} +{{% tab name="Web UI" %}} + +1. Open the [**DATA** page](https://app.viam.com/data/view). + +1. Navigate to the **ALL DATA** tab. + +1. Use the checkbox in the upper left of each image to select labeled images. + +1. Click the **Add to dataset** button, select a dataset, and click the **Add ... images** button to add the selected images to the dataset. + +{{% /tab %}} +{{% tab name="CLI" %}} + +Use the Viam CLI to filter images by label and add the filtered images to a dataset: + +1. First, [create a dataset](#create-a-dataset), if you haven't already. + +1. If you just created a dataset, use the dataset ID output by the creation command. + If your dataset already exists, run the following command to get a list of dataset names and corresponding IDs: + + ```sh {class="command-line" data-prompt="$"} + viam dataset list + ``` + +1. Run the following [command](/dev/tools/cli/#dataset) to add all images labeled with a subset of tags to the dataset, replacing the `` placeholder with the dataset ID output by the command in the previous step: + + ```sh {class="command-line" data-prompt="$"} + viam dataset data add filter --dataset-id= --tags=red_star,blue_square + ``` + +{{% /tab %}} +{{% tab name="SDK" %}} + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +import asyncio +from PIL import Image +from viam.app.app_client import AppClient +from viam.app.data_client import DataClient + +MACHINE_PART_ID = "your-machine-part-id-here" +DATASET_ID = "your-dataset-id-here" +API_KEY_ID = "your-api-key-id" +API_KEY = "your-api-key" + +async def add_image_to_dataset(): + """Add image to Viam dataset using Python SDK""" + + # Initialize app client + app_client = AppClient.create_from_api_key( + api_key_id=API_KEY_ID, + api_key=API_KEY + ) + + try: + # Get data client + data_client = app_client.data_client + + # Load image file + image_path = "sample_image.jpg" + with open(image_path, "rb") as image_file: + image_data = image_file.read() + + # Upload image to Viam cloud + file_upload_metadata = await data_client.file_upload_from_bytes( + part_id=MACHINE_PART_ID, + data=image_data, + component_type="camera", + component_name="camera-1", + file_name="training_sample.jpg", + file_extension="jpg" + ) + + file_id = file_upload_metadata.file_id + print(f"Image uploaded successfully. File ID: {file_id}") + + # Add uploaded file to dataset + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[file_id], + dataset_id=DATASET_ID + ) + + print(f"Image added to dataset {DATASET_ID} successfully") + + except Exception as error: + print(f"Operation failed: {error}") + raise + + finally: + app_client.close() + +# Execute function +if __name__ == "__main__": + asyncio.run(add_image_to_dataset()) + +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package main + +import ( + "context" + "fmt" + "os" + "time" + + "go.viam.com/rdk/app/appclient" + "go.viam.com/rdk/services/datamanager/app" +) + +const ( + MachinePartID = "your-machine-part-id-here" + DatasetID = "your-dataset-id-here" + APIKeyID = "your-api-key-id" + APIKey = "your-api-key" +) + +func addImageToDataset() error { + ctx := context.Background() + + // Create app client + client, err := appclient.New(ctx, appclient.Config{ + KeyID: APIKeyID, + Key: APIKey, + }) + if err != nil { + return fmt.Errorf("failed to create app client: %w", err) + } + defer client.Close() + + // Get data client + dataClient := client.DataClient + + // Read image file + imagePath := "sample_image.jpg" + imageData, err := os.ReadFile(imagePath) + if err != nil { + return fmt.Errorf("failed to read image file: %w", err) + } + + // Prepare upload options + componentType := "camera" + componentName := "camera-1" + fileName := fmt.Sprintf("training_sample_%d.jpg", time.Now().Unix()) + fileExtension := "jpg" + + uploadOptions := app.FileUploadOptions{ + ComponentType: &componentType, + ComponentName: &componentName, + FileName: &fileName, + FileExtension: &fileExtension, + } + + // Upload image + fileID, err := dataClient.FileUploadFromBytes( + ctx, + MachinePartID, + imageData, + &uploadOptions, + ) + if err != nil { + return fmt.Errorf("failed to upload image: %w", err) + } + + fmt.Printf("Image uploaded successfully. File ID: %s\n", fileID) + + // Add file to dataset + err = dataClient.AddBinaryDataToDatasetByIDs( + ctx, + []string{fileID}, + DatasetID, + ) + if err != nil { + return fmt.Errorf("failed to add image to dataset: %w", err) + } + + fmt.Printf("Image added to dataset %s successfully\n", DatasetID) + return nil +} + +func main() { + if err := addImageToDataset(); err != nil { + fmt.Printf("Operation failed: %v\n", err) + os.Exit(1) + } +} + +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +import { createAppClient } from '@viamrobotics/app-client'; +import { promises as fs } from 'fs'; + +const MACHINE_PART_ID = 'your-machine-part-id-here'; +const DATASET_ID = 'your-dataset-id-here'; +const API_KEY_ID = 'your-api-key-id'; +const API_KEY = 'your-api-key'; + +async function addImageToDataset(): Promise { + // Create app client + const appClient = createAppClient({ + apiKeyId: API_KEY_ID, + apiKey: API_KEY, + }); + + try { + // Get data client + const dataClient = appClient.dataClient(); + + // Read image file + const imagePath = 'sample_image.jpg'; + const imageBuffer = await fs.readFile(imagePath); + const imageData = new Uint8Array(imageBuffer); + + // Upload image + const uploadResponse = await dataClient.fileUploadFromBytes({ + partId: MACHINE_PART_ID, + data: imageData, + componentType: 'camera', + componentName: 'camera-1', + fileName: `training_sample_${Date.now()}.jpg`, + fileExtension: 'jpg', + }); + + const fileId = uploadResponse.fileId; + console.log(`Image uploaded successfully. File ID: ${fileId}`); + + // Add file to dataset + await dataClient.addBinaryDataToDatasetByIds({ + binaryIds: [fileId], + datasetId: DATASET_ID, + }); + + console.log(`Image added to dataset ${DATASET_ID} successfully`); + + } catch (error) { + console.error(`Operation failed: ${error}`); + throw error; + } finally { + await appClient.close(); + } +} + +// Execute function +addImageToDataset().catch((error) => { + console.error('Failed to add image to dataset:', error); + process.exit(1); +}); + +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'dart:io'; +import 'dart:typed_data'; +import 'package:viam_sdk/viam_sdk.dart'; + +const String machinePartId = 'your-machine-part-id-here'; +const String datasetId = 'your-dataset-id-here'; +const String apiKeyId = 'your-api-key-id'; +const String apiKey = 'your-api-key'; + +Future addImageToDataset() async { + AppClient? appClient; + + try { + // Create app client + appClient = await AppClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + + // Get data client + final dataClient = appClient.dataClient; + + // Read image file + const imagePath = 'sample_image.jpg'; + final imageFile = File(imagePath); + final imageBytes = await imageFile.readAsBytes(); + + // Upload image + final uploadOptions = FileUploadOptions( + componentType: 'camera', + componentName: 'camera-1', + fileName: 'training_sample_${DateTime.now().millisecondsSinceEpoch}.jpg', + fileExtension: 'jpg', + ); + + final fileId = await dataClient.fileUploadFromBytes( + machinePartId, + imageBytes, + uploadOptions, + ); + + print('Image uploaded successfully. File ID: $fileId'); + + // Add file to dataset + await dataClient.addBinaryDataToDatasetByIds( + [fileId], + datasetId, + ); + + print('Image added to dataset $datasetId successfully'); + + } catch (error) { + print('Operation failed: $error'); + rethrow; + } finally { + await appClient?.close(); + } +} + +void main() async { + try { + await addImageToDataset(); + } catch (error) { + print('Failed to add image to dataset: $error'); + exit(1); + } +} +``` + +{{% /tab %}} +{{< /tabs >}} + +{{% /tab %}} +{{< /tabs >}} + + +## Add all images captured by a specific machine to a dataset + +The following script adds all images captured from a certain machine to a new dataset: + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +import asyncio +from typing import List, Optional + +from viam.rpc.dial import DialOptions, Credentials +from viam.app.viam_client import ViamClient +from viam.utils import create_filter + +# Configuration constants – replace with your actual values +DATASET_NAME = "" # a unique, new name for the dataset you want to create +ORG_ID = "" # your organization ID, find in your organization settings +PART_ID = "" # ID of machine that captured target images, find in machine config +API_KEY = "" # API key, find or create in your organization settings +API_KEY_ID = "" # API key ID, find or create in your organization settings + +# Adjust the maximum number of images to add to the dataset +MAX_MATCHES = 500 + +async def connect() -> ViamClient: + """Establish a connection to the Viam client using API credentials.""" + dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, + ) + return await ViamClient.create_from_dial_options(dial_options) + + +async def fetch_binary_data_ids(data_client, part_id: str) -> List[str]: + """Fetch binary data metadata and return a list of BinaryData objects.""" + data_filter = create_filter(part_id=part_id) + all_matches = [] + last: Optional[str] = None + + print("Getting data for part...") + + while len(all_matches) < MAX_MATCHES: + print("Fetching more data...") + data, _, last = await data_client.binary_data_by_filter( + data_filter, + limit=50, + last=last, + include_binary_data=False, + ) + if not data: + break + all_matches.extend(data) + + return all_matches + + +async def main() -> int: + """Main execution function.""" + viam_client = await connect() + data_client = viam_client.data_client + + matching_data = await fetch_binary_data_ids(data_client, PART_ID) + + print("Creating dataset...") + + try: + dataset_id = await data_client.create_dataset( + name=DATASET_NAME, + organization_id=ORG_ID, + ) + print(f"Created dataset: {dataset_id}") + except Exception as e: + print("Error creating dataset. It may already exist.") + print("See: https://app.viam.com/data/datasets") + print(f"Exception: {e}") + return 1 + + print("Adding data to dataset...") + + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[obj.metadata.binary_data_id for obj in matching_data], + dataset_id=dataset_id + ) + + print("Added files to dataset.") + print(f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}") + + viam_client.close() + return 0 + + +if __name__ == "__main__": + asyncio.run(main()) +``` +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package main + +import ( + "context" + "fmt" + "log" + + "go.viam.com/rdk/rpc" + "go.viam.com/utils" + "go.viam.com/rdk/app" + "go.viam.com/rdk/app/data" +) + +// Configuration constants - replace with your actual values +const ( + DATASET_NAME = "" // a unique, new name for the dataset you want to create + ORG_ID = "" // your organization ID, find in your organization settings + PART_ID = "" // ID of machine that captured target images, find in machine config + API_KEY = "" // API key, find or create in your organization settings + API_KEY_ID = "" // API key ID, find or create in your organization settings + MAX_MATCHES = 500 +) + +func connect(ctx context.Context) (*app.ViamClient, error) { + client, err := app.NewViamClientWithAPIKey(ctx, API_KEY, API_KEY_ID) + if err != nil { + return nil, fmt.Errorf("failed to create client: %w", err) + } + return client, nil +} + +func fetchBinaryDataIDs(ctx context.Context, dataClient data.DataServiceClient, partID string) ([]*data.BinaryData, error) { + filter := &data.Filter{ + PartId: partID, + } + + var allMatches []*data.BinaryData + var last string + + fmt.Println("Getting data for part...") + + for len(allMatches) < MAX_MATCHES { + fmt.Println("Fetching more data...") + + resp, err := dataClient.BinaryDataByFilter(ctx, &data.BinaryDataByFilterRequest{ + DataRequest: &data.DataRequest{ + Filter: filter, + Limit: 50, + Last: last, + IncludeBinaryData: false, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch binary data: %w", err) + } + + if len(resp.Data) == 0 { + break + } + + allMatches = append(allMatches, resp.Data...) + last = resp.Last + } + + return allMatches, nil +} + +func main() { + ctx := context.Background() + + viamClient, err := connect(ctx) + if err != nil { + log.Fatalf("Connection failed: %v", err) + } + defer viamClient.Close() + + dataClient := viamClient.DataClient + + matchingData, err := fetchBinaryDataIDs(ctx, dataClient, PART_ID) + if err != nil { + log.Fatalf("Failed to fetch binary data: %v", err) + } + + fmt.Println("Creating dataset...") + + datasetResp, err := dataClient.CreateDataset(ctx, &data.CreateDatasetRequest{ + Name: DATASET_NAME, + OrganizationId: ORG_ID, + }) + if err != nil { + fmt.Println("Error creating dataset. It may already exist.") + fmt.Println("See: https://app.viam.com/data/datasets") + fmt.Printf("Exception: %v\n", err) + return + } + + datasetID := datasetResp.Id + fmt.Printf("Created dataset: %s\n", datasetID) + + fmt.Println("Adding data to dataset...") + + var binaryIDs []string + for _, obj := range matchingData { + binaryIDs = append(binaryIDs, obj.Metadata.Id) + } + + _, err = dataClient.AddBinaryDataToDatasetByIds(ctx, &data.AddBinaryDataToDatasetByIdsRequest{ + BinaryIds: binaryIDs, + DatasetId: datasetID, + }) + if err != nil { + log.Fatalf("Failed to add binary data to dataset: %v", err) + } + + fmt.Println("Added files to dataset.") + fmt.Printf("See dataset: https://app.viam.com/data/datasets?id=%s\n", datasetID) +} + +``` + +{{% /tab %}} +{{% tab name="Typescript" %}} + +```typescript +import { ViamClient, createViamClient } from '@viamrobotics/sdk'; +import { DataServiceClient } from '@viamrobotics/sdk/dist/gen/app/data/v1/data_pb_service'; +import { + BinaryDataByFilterRequest, + CreateDatasetRequest, + AddBinaryDataToDatasetByIdsRequest, + Filter, + DataRequest +} from '@viamrobotics/sdk/dist/gen/app/data/v1/data_pb'; + +// Configuration constants - replace with your actual values +const DATASET_NAME = ""; // a unique, new name for the dataset you want to create +const ORG_ID = ""; // your organization ID, find in your organization settings +const PART_ID = ""; // ID of machine that captured target images, find in machine config +const API_KEY = ""; // API key, find or create in your organization settings +const API_KEY_ID = ""; // API key ID, find or create in your organization settings +const MAX_MATCHES = 500; + +async function connect(): Promise { + return await createViamClient({ + credential: { + type: 'api-key', + authEntity: API_KEY_ID, + payload: API_KEY, + }, + }); +} + +async function fetchBinaryDataIds(dataClient: DataServiceClient, partId: string): Promise { + const filter = new Filter(); + filter.setPartId(partId); + + const allMatches: any[] = []; + let last = ""; + + console.log("Getting data for part..."); + + while (allMatches.length < MAX_MATCHES) { + console.log("Fetching more data..."); + + const dataRequest = new DataRequest(); + dataRequest.setFilter(filter); + dataRequest.setLimit(50); + dataRequest.setLast(last); + dataRequest.setIncludeBinaryData(false); + + const request = new BinaryDataByFilterRequest(); + request.setDataRequest(dataRequest); + + const response = await dataClient.binaryDataByFilter(request); + const data = response.getDataList(); + + if (data.length === 0) { + break; + } + + allMatches.push(...data); + last = response.getLast(); + } + + return allMatches; +} + +async function main(): Promise { + const viamClient = await connect(); + const dataClient = viamClient.dataClient; + + const matchingData = await fetchBinaryDataIds(dataClient, PART_ID); + + console.log("Creating dataset..."); + + try { + const createRequest = new CreateDatasetRequest(); + createRequest.setName(DATASET_NAME); + createRequest.setOrganizationId(ORG_ID); + + const datasetResponse = await dataClient.createDataset(createRequest); + const datasetId = datasetResponse.getId(); + console.log(`Created dataset: ${datasetId}`); + + console.log("Adding data to dataset..."); + + const binaryIds = matchingData.map(obj => obj.getMetadata()?.getId() || ""); + + const addRequest = new AddBinaryDataToDatasetByIdsRequest(); + addRequest.setBinaryIdsList(binaryIds); + addRequest.setDatasetId(datasetId); + + await dataClient.addBinaryDataToDatasetByIds(addRequest); + + console.log("Added files to dataset."); + console.log(`See dataset: https://app.viam.com/data/datasets?id=${datasetId}`); + + viamClient.disconnect(); + return 0; + + } catch (error) { + console.log("Error creating dataset. It may already exist."); + console.log("See: https://app.viam.com/data/datasets"); + console.log(`Exception: ${error}`); + viamClient.disconnect(); + return 1; + } +} + +if (require.main === module) { + main().then(code => process.exit(code)); +} + +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'package:viam_sdk/viam_sdk.dart'; + +// Configuration constants - replace with your actual values +const String datasetName = ""; // a unique, new name for the dataset you want to create +const String orgId = ""; // your organization ID, find in your organization settings +const String partId = ""; // ID of machine that captured target images, find in machine config +const String apiKey = ""; // API key, find or create in your organization settings +const String apiKeyId = ""; // API key ID, find or create in your organization settings +const int maxMatches = 500; + +Future connect() async { + return await ViamClient.withApiKey( + apiKey: apiKey, + apiKeyId: apiKeyId, + ); +} + +Future> fetchBinaryDataIds( + DataClient dataClient, String partId) async { + final filter = Filter(partId: partId); + final List allMatches = []; + String? last; + + print("Getting data for part..."); + + while (allMatches.length < maxMatches) { + print("Fetching more data..."); + + final response = await dataClient.binaryDataByFilter( + filter: filter, + limit: 50, + last: last, + includeBinaryData: false, + ); + + if (response.data.isEmpty) { + break; + } + + allMatches.addAll(response.data); + last = response.last; + } + + return allMatches; +} + +Future main() async { + final viamClient = await connect(); + final dataClient = viamClient.dataClient; + + final matchingData = await fetchBinaryDataIds(dataClient, partId); + + print("Creating dataset..."); + + try { + final datasetId = await dataClient.createDataset( + name: datasetName, + organizationId: orgId, + ); + print("Created dataset: $datasetId"); + + print("Adding data to dataset..."); + + final binaryIds = matchingData + .map((obj) => obj.metadata.binaryDataId) + .toList(); + + await dataClient.addBinaryDataToDatasetByIds( + binaryIds: binaryIds, + datasetId: datasetId, + ); + + print("Added files to dataset."); + print("See dataset: https://app.viam.com/data/datasets?id=$datasetId"); + + viamClient.close(); + return 0; + + } catch (error) { + print("Error creating dataset. It may already exist."); + print("See: https://app.viam.com/data/datasets"); + print("Exception: $error"); + viamClient.close(); + return 1; + } +} +``` + +{{% /tab %}} +{{< /tabs >}} diff --git a/docs/data-ai/ai/advanced/upload-external-data.md b/docs/data-ai/train/upload-external-data.md similarity index 97% rename from docs/data-ai/ai/advanced/upload-external-data.md rename to docs/data-ai/train/upload-external-data.md index 56ac6d7307..d25af1dc12 100644 --- a/docs/data-ai/ai/advanced/upload-external-data.md +++ b/docs/data-ai/train/upload-external-data.md @@ -2,7 +2,7 @@ linkTitle: "Upload external data" title: "Upload external data for training" images: ["/services/icons/data-folder.svg"] -weight: 20 +weight: 60 layout: "docs" type: "docs" languages: ["python"] @@ -11,6 +11,7 @@ aliases: - /data/upload/ - /services/data/upload/ - /how-tos/upload-data/ + - /data-ai/ai/advanced/upload-external-data/ date: "2024-12-04" description: "Upload data to Viam from your local computer or mobile device using the data client API, Viam CLI, or Viam mobile app." prev: "/data-ai/ai/act/" @@ -135,9 +136,7 @@ Create a Python script and use the `file_upload_from_path` method to upload your {{< tabs >}} {{< tab name="Upload a single file" >}} -To upload just one file, make a call to [`file_upload_from_path`](/dev/reference/apis/data-client/#fileuploadfrompath). - -{{< expand "Click this to see example code" >}} +To upload just one file, make a call to [`file_upload_from_path`](/dev/reference/apis/data-client/#fileuploadfrompath): ```python {class="line-numbers linkable-line-numbers"} import asyncio @@ -180,15 +179,11 @@ if __name__ == "__main__": asyncio.run(main()) ``` -{{< /expand >}} - {{% /tab %}} {{< tab name="Upload all files in a directory" >}} To upload all the files in a directory, you can use the [`file_upload_from_path`](/dev/reference/apis/data-client/#fileuploadfrompath) method inside a `for` loop. -{{< expand "Click this to see example code" >}} - ```python {class="line-numbers linkable-line-numbers"} import asyncio import os @@ -232,8 +227,6 @@ if __name__ == "__main__": asyncio.run(main()) ``` -{{< /expand >}} - {{% /tab %}} {{< /tabs >}} @@ -296,5 +289,5 @@ However, the uploaded images will not be associated with a component or method. ## Next steps -Now that you have a batch of data uploaded, you can [train an ML model](/data-ai/ai/train-tflite/) on it. -Or, if you want to collect and upload data _not_ in a batch, see [Create a dataset](/data-ai/ai/create-dataset/). +Now that you have a batch of data uploaded, you can [train an ML model](/data-ai/train/train-tflite/) on it. +Or, if you want to collect and upload data _not_ in a batch, see [Create a dataset](/data-ai/train/create-dataset/). diff --git a/docs/dev/_index.md b/docs/dev/_index.md index 5e7a3bff4c..96d513ab79 100644 --- a/docs/dev/_index.md +++ b/docs/dev/_index.md @@ -753,7 +753,7 @@ job_metadata = await ml_training_client.get_training_job( Build machine learning models based on your machines' data any time using the ML training client API -[Train and deploy ML models →](/data-ai/ai/train-tflite/) +[Train and deploy ML models →](/data-ai/train/train-tflite/) diff --git a/docs/dev/reference/changelog.md b/docs/dev/reference/changelog.md index 2608031b5c..e612621c08 100644 --- a/docs/dev/reference/changelog.md +++ b/docs/dev/reference/changelog.md @@ -448,7 +448,7 @@ For more information, see [Supported components and services](/data-ai/capture-d {{% changelog date="2024-08-01" color="added" title="Create custom training scripts" %}} You can now upload custom training scripts to the Viam Registry and use them to train machine learning models. -For more information, see [Create custom training scripts](/data-ai/ai/train/). +For more information, see [Create custom training scripts](/data-ai/train/train/). {{% /changelog %}} @@ -678,7 +678,7 @@ For more information, see [Query Data with SQL or MQL](/data-ai/data/query/). {{% changelog date="2023-11-30" color="changed" title="Model training from datasets" %}} -To make it easier to iterate while training machine learning models from image data, you now train models from [datasets](/data-ai/ai/create-dataset/). +To make it easier to iterate while training machine learning models from image data, you now train models from [datasets](/data-ai/train/create-dataset/). {{% /changelog %}} @@ -854,7 +854,7 @@ To better control gantries with Viam, you can now: {{% changelog date="2023-06-30" color="improved" title="Optimized Viam-trained object detection models" %}} -This update for TFlite object detection models [trained with the machine learning service](/data-ai/ai/train-tflite/) brings significant improvements, including: +This update for TFlite object detection models [trained with the machine learning service](/data-ai/train/train-tflite/) brings significant improvements, including: - 76% faster model inference for camera streams - 64% quicker model training for object detection @@ -870,7 +870,7 @@ The beta release of the [TypeScript SDK](https://github.com/viamrobotics/viam-ty {{% changelog date="2023-05-31" color="added" title="Train object detection ML models" %}} -You now have the capability to directly [train a TFlite object detection models](/data-ai/ai/train-tflite/). +You now have the capability to directly [train a TFlite object detection models](/data-ai/train/train-tflite/). This update allows you to: @@ -1293,9 +1293,9 @@ You will need to first register the machine learning model file with the [ML mod {{% changelog date="2023-03-31" color="added" title="Machine learning for image classification models" %}} -You can now [train](/data-ai/ai/train-tflite/) and [deploy](/data-ai/ai/deploy/) image classification models with the [data management service](/data-ai/capture-data/capture-sync/) and use your machine's image data directly within Viam. +You can now [train](/data-ai/train/train-tflite/) and [deploy](/data-ai/ai/deploy/) image classification models with the [data management service](/data-ai/capture-data/capture-sync/) and use your machine's image data directly within Viam. Additionally, you can upload and use existing [machine learning models](/data-ai/ai/deploy/#deploy-your-ml-model-on-an-ml-model-service) with your machines. -For more information on using data synced to the cloud to train machine learning models, read [train a TFlite](/data-ai/ai/train-tflite/) or [another model](/data-ai/ai/train/). +For more information on using data synced to the cloud to train machine learning models, read [train a TFlite](/data-ai/train/train-tflite/) or [another model](/data-ai/train/train/). {{% /changelog %}} diff --git a/docs/dev/reference/glossary/ml.md b/docs/dev/reference/glossary/ml.md index 717bb0a280..c1b6f16563 100644 --- a/docs/dev/reference/glossary/ml.md +++ b/docs/dev/reference/glossary/ml.md @@ -9,4 +9,4 @@ type: "page" ML stands for machine learning, a field of artificial intelligence that focuses on building systems that can learn from and make decisions based on data. -Viam provides tools for [training ML models](/data-ai/ai/train/), [deploying them to machines](/data-ai/ai/deploy/), [running inference](/data-ai/ai/run-inference/), and [interpret visual data from cameras](/data-ai/ai/alert/) to enable intelligent behavior in robotic systems. +Viam provides tools for [training ML models](/data-ai/train/train/), [deploying them to machines](/data-ai/ai/deploy/), [running inference](/data-ai/ai/run-inference/), and [interpret visual data from cameras](/data-ai/ai/alert/) to enable intelligent behavior in robotic systems. diff --git a/docs/dev/tools/cli.md b/docs/dev/tools/cli.md index ad39953c5c..ba3e32c687 100644 --- a/docs/dev/tools/cli.md +++ b/docs/dev/tools/cli.md @@ -399,7 +399,7 @@ The **Binary Data ID** is shown under the **DETAILS** subtab that appears on the You cannot use filter arguments such as `--start` or `--end` with the `ids` argument. -See [Create a dataset](/data-ai/ai/create-dataset/) for more information. +See [Create a dataset](/data-ai/train/create-dataset/) for more information. ##### Using the `filter` argument @@ -430,7 +430,7 @@ Removing the `viam data export` string, you can use the same filter parameters ( You cannot use the `--binary-data-ids` argument when using `filter`. -See [Create a dataset](/data-ai/ai/create-dataset/) for more information. +See [Create a dataset](/data-ai/train/create-dataset/) for more information. ### `data` @@ -1365,7 +1365,7 @@ You can set a default profile by using the `VIAM_CLI_PROFILE_NAME` environment v ### `training-script` -Manage training scripts for [custom ML training](/data-ai/ai/train/). +Manage training scripts for [custom ML training](/data-ai/train/train/). ```sh {class="command-line" data-prompt="$"} viam training-script upload --framework= --org-id= --path= --script-name= --type= diff --git a/docs/manage/troubleshoot/teleoperate/default-interface.md b/docs/manage/troubleshoot/teleoperate/default-interface.md index 90f300b86d..2cd5b36ecc 100644 --- a/docs/manage/troubleshoot/teleoperate/default-interface.md +++ b/docs/manage/troubleshoot/teleoperate/default-interface.md @@ -45,7 +45,7 @@ Additionally, the app allows you to: - see if your machines are online - [view a machine's logs](/manage/troubleshoot/troubleshoot/#check-logs) -- [upload images from your phone to the cloud](/data-ai/ai/advanced/upload-external-data/#upload-images-with-the-viam-mobile-app) +- [upload images from your phone to the cloud](/data-ai/train/upload-external-data/#upload-images-with-the-viam-mobile-app) - [invite people to collaborate with you and modify access](/manage/troubleshoot/teleoperate/default-interface/#viam-mobile-app)
diff --git a/docs/operate/get-started/supported-hardware/_index.md b/docs/operate/get-started/supported-hardware/_index.md index a48d2335e5..fb6366c102 100644 --- a/docs/operate/get-started/supported-hardware/_index.md +++ b/docs/operate/get-started/supported-hardware/_index.md @@ -219,7 +219,7 @@ If you have other hardware you need to integrate with a custom module, continue If you have configured all your hardware, you can do a variety of things with your machine: - [Capture data from your machines](/data-ai/capture-data/capture-sync/) -- [Create a dataset](/data-ai/ai/create-dataset/) and [train an AI model](/data-ai/ai/train-tflite/) +- [Create a dataset](/data-ai/train/create-dataset/) and [train an AI model](/data-ai/train/train-tflite/) - [Write an app](/operate/control/web-app/) to interact with your machines using any of the Viam SDKs - [Deploy control logic to run directly on your machines](/manage/software/control-logic/) - [Share the configuration across multiple machines](/manage/fleet/reuse-configuration/) diff --git a/docs/operate/reference/advanced-modules/_index.md b/docs/operate/reference/advanced-modules/_index.md index 4468555a4a..2cd61420e5 100644 --- a/docs/operate/reference/advanced-modules/_index.md +++ b/docs/operate/reference/advanced-modules/_index.md @@ -52,6 +52,6 @@ If you need to package and deploy a module using Docker, for example if your mod ## Design a custom ML model -When working with the [ML model service](/dev/reference/apis/services/ml/), you can deploy an [existing model](/data-ai/ai/deploy/) or [train your own model](/data-ai/ai/train/). +When working with the [ML model service](/dev/reference/apis/services/ml/), you can deploy an [existing model](/data-ai/ai/deploy/) or [train your own model](/data-ai/train/train/). However, if you are writing your own {{< glossary_tooltip term_id="module" text="module" >}} that uses the ML model service together with the [vision service](/dev/reference/apis/services/vision/), you can also [design your own ML model](/data-ai/reference/mlmodel-design/) to better match your specific use case. diff --git a/docs/operate/reference/services/vision/mlmodel.md b/docs/operate/reference/services/vision/mlmodel.md index 46b61ba764..7aed2ed7a0 100644 --- a/docs/operate/reference/services/vision/mlmodel.md +++ b/docs/operate/reference/services/vision/mlmodel.md @@ -29,7 +29,7 @@ Before configuring your `mlmodel` detector or classifier, you need to:

1. Train or upload an ML model

-You can add an [existing model](/data-ai/ai/deploy/#deploy-your-ml-model-on-an-ml-model-service) or [train a TFlite](/data-ai/ai/train-tflite/) or [another model](/data-ai/ai/train/) for object detection and classification using your data in the [Viam Cloud](/data-ai/capture-data/capture-sync/). +You can add an [existing model](/data-ai/ai/deploy/#deploy-your-ml-model-on-an-ml-model-service) or [train a TFlite](/data-ai/train/train-tflite/) or [another model](/data-ai/train/train/) for object detection and classification using your data in the [Viam Cloud](/data-ai/capture-data/capture-sync/). {{% /manualcard %}} {{% manualcard %}} @@ -146,7 +146,7 @@ Both the `mlmodel` detector and classifier require that the input and output ten - The _input tensor_ must be named `image` - The _output tensor_ must be named `probability` -If you [trained a TFlite ML model using Viam](/data-ai/ai/train-tflite/), your `mlmodel` tensors are already named in this fashion, and you can proceed to [test your detector or classifier](#test-your-detector-or-classifier). +If you [trained a TFlite ML model using Viam](/data-ai/train/train-tflite/), your `mlmodel` tensors are already named in this fashion, and you can proceed to [test your detector or classifier](#test-your-detector-or-classifier). However, if you uploaded your own ML model, or are using one from the [registry](https://app.viam.com/registry), you may need to remap your tensor names to meet this requirement, and should follow the instructions to [remap tensor names](#remap-tensor-names). #### Remap tensor names diff --git a/docs/tutorials/projects/helmet.md b/docs/tutorials/projects/helmet.md index 9991909aa4..d03173195e 100644 --- a/docs/tutorials/projects/helmet.md +++ b/docs/tutorials/projects/helmet.md @@ -460,10 +460,10 @@ Here are some ways you could expand on this project: - Change your cloud function to send a different kind of notification, or trigger some other action. For an example demonstrating how to configure text notifications, see the [Detect a Person and Send a Photo tutorial](/tutorials/projects/send-security-photo/). -- Use a different existing model or [train your own](/data-ai/ai/train-tflite/), to detect and send notifications about something else such as [forklifts](https://huggingface.co/keremberke/yolov8m-forklift-detection) appearing in your camera stream. +- Use a different existing model or [train your own](/data-ai/train/train-tflite/), to detect and send notifications about something else such as [forklifts](https://huggingface.co/keremberke/yolov8m-forklift-detection) appearing in your camera stream. {{< cards >}} {{% card link="/tutorials/projects/send-security-photo/" %}} -{{% card link="/data-ai/ai/train-tflite/" %}} +{{% card link="/data-ai/train/train-tflite/" %}} {{% card link="/tutorials/services/navigate-with-rover-base/" %}} {{< /cards >}} diff --git a/docs/tutorials/projects/send-security-photo.md b/docs/tutorials/projects/send-security-photo.md index 3c880fd93d..19495ac360 100644 --- a/docs/tutorials/projects/send-security-photo.md +++ b/docs/tutorials/projects/send-security-photo.md @@ -88,7 +88,7 @@ This tutorial uses a pre-trained Machine Learning model from the Viam Registry c The model can detect a variety of things, including `Persons`. You can see a full list of what the model can detect in [labels.txt](https://github.com/viam-labs/devrel-demos/raw/main/Light%20up%20bot/labels.txt) file. -If you want to train your own model instead, follow the instructions to [train a TFlite](/data-ai/ai/train-tflite/) or [another model](/data-ai/ai/train/). +If you want to train your own model instead, follow the instructions to [train a TFlite](/data-ai/train/train-tflite/) or [another model](/data-ai/train/train/). 1. **Configure the ML model service** diff --git a/docs/tutorials/projects/verification-system.md b/docs/tutorials/projects/verification-system.md index 96203f28ad..b85b4f4e19 100644 --- a/docs/tutorials/projects/verification-system.md +++ b/docs/tutorials/projects/verification-system.md @@ -172,7 +172,7 @@ Then, train a new model using that model: {{}} 1. When you have created bounding boxes for all `person` objects in the image, click the right arrow key to navigate to the next image. Repeat the process for each image in your dataset, drawing bounding boxes for every person in every image. -1. [Train a TFlite model on your dataset](/data-ai/ai/train-tflite/). +1. [Train a TFlite model on your dataset](/data-ai/train/train-tflite/). Give it the name `"persondetect"`, and select **Object Detection** as the **Model Type**. 1. [Deploy the model](/data-ai/ai/deploy/) to your machine so it can be used by other services, such as the vision service. diff --git a/layouts/docs/tutorials.html b/layouts/docs/tutorials.html index e659b3c1df..efe18326de 100644 --- a/layouts/docs/tutorials.html +++ b/layouts/docs/tutorials.html @@ -92,7 +92,7 @@

Javascript

{{ partial "tutorialcard-no-js.html" (dict "link" "/tutorials/services/constrain-motion/") }} {{ partial "tutorialcard-no-js.html" (dict "link" "/tutorials/services/color-detection-scuttle/") }} {{ partial "tutorialcard-no-js.html" (dict "link" "/tutorials/services/webcam-line-follower-robot/") }} - {{ partial "tutorialcard-no-js.html" (dict "link" "/data-ai/ai/train-tflite/") }} + {{ partial "tutorialcard-no-js.html" (dict "link" "/data-ai/train/train-tflite/") }} diff --git a/static/include/app/apis/generated/mltraining.md b/static/include/app/apis/generated/mltraining.md index db31faf672..5830b1d0af 100644 --- a/static/include/app/apis/generated/mltraining.md +++ b/static/include/app/apis/generated/mltraining.md @@ -72,7 +72,7 @@ For more information, see the [TypeScript SDK Docs](https://ts.viam.dev/classes/ ### SubmitCustomTrainingJob Submit a training job from a custom training script. -Follow the guide to [Train a Model with a Custom Python Training Script](/data-ai/ai/train/). +Follow the guide to [Train a Model with a Custom Python Training Script](/data-ai/train/train/). {{< tabs >}} {{% tab name="Python" %}} diff --git a/static/include/app/apis/overrides/protos/mltraining.SubmitCustomTrainingJob.md b/static/include/app/apis/overrides/protos/mltraining.SubmitCustomTrainingJob.md index fd12ae964b..c7f237fea0 100644 --- a/static/include/app/apis/overrides/protos/mltraining.SubmitCustomTrainingJob.md +++ b/static/include/app/apis/overrides/protos/mltraining.SubmitCustomTrainingJob.md @@ -1,2 +1,2 @@ Submit a training job from a custom training script. -Follow the guide to [Train a Model with a Custom Python Training Script](/data-ai/ai/train/). +Follow the guide to [Train a Model with a Custom Python Training Script](/data-ai/train/train/). From ce48bc12f67b79bf56f561f530f3f562db928f9b Mon Sep 17 00:00:00 2001 From: nathan contino Date: Wed, 25 Jun 2025 17:08:59 -0400 Subject: [PATCH 02/29] Lint code samples --- docs/data-ai/train/annotate-images.md | 60 +++++---- docs/data-ai/train/capture-images.md | 63 +++++----- docs/data-ai/train/create-dataset.md | 18 +-- docs/data-ai/train/update-dataset.md | 173 +++++++++++++------------- 4 files changed, 156 insertions(+), 158 deletions(-) diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/annotate-images.md index f91b41b73e..74a80fe01c 100644 --- a/docs/data-ai/train/annotate-images.md +++ b/docs/data-ai/train/annotate-images.md @@ -165,33 +165,30 @@ const myDetector = new VisionClient(client, ""); const dataClient = client.dataClient; // Get the captured data for a camera -const result = await myDetector.captureAllFromCamera( - "", - { - returnImage: true, - returnDetections: true, - } -); +const result = await myDetector.captureAllFromCamera("", { + returnImage: true, + returnDetections: true, +}); const image = result.image; const detections = result.detections; const tags = ["tag1", "tag2"]; const myFilter = createFilter({ - componentName: "camera-1", - organizationIds: [""] + componentName: "camera-1", + organizationIds: [""], }); const binaryResult = await dataClient.binaryDataByFilter({ - filter: myFilter, - limit: 20, - includeBinaryData: false + filter: myFilter, + limit: 20, + includeBinaryData: false, }); const myIds: string[] = []; for (const obj of binaryResult.binaryMetadata) { - myIds.push(obj.metadata.binaryDataId); + myIds.push(obj.metadata.binaryDataId); } await dataClient.addTagsToBinaryDataByIds(tags, myIds); @@ -375,7 +372,7 @@ for _, detection := range detections { BinaryID: "", Label: detection.Label, XMinNormalized: detection.BoundingBox.Min.X, - YMinNormalized: detection.BoundingBox.Min.Y, + YMinNormalized: detection.BoundingBox.Min.Y, XMaxNormalized: detection.BoundingBox.Max.X, YMaxNormalized: detection.BoundingBox.Max.Y, }) @@ -393,28 +390,27 @@ const myDetector = new VisionClient(client, ""); const dataClient = client.dataClient; // Get the captured data for a camera -const result = await myDetector.captureAllFromCamera( - "", - { - returnImage: true, - returnDetections: true, - } -); +const result = await myDetector.captureAllFromCamera("", { + returnImage: true, + returnDetections: true, +}); const image = result.image; const detections = result.detections; // Process each detection and add bounding boxes for (const detection of detections) { - const bboxId = await dataClient.addBoundingBoxToImageById({ - binaryId: "", - label: detection.className, - xMinNormalized: detection.boundingBox.xMin, - yMinNormalized: detection.boundingBox.yMin, - xMaxNormalized: detection.boundingBox.xMax, - yMaxNormalized: detection.boundingBox.yMax - }); - - console.log(`Added bounding box ID: ${bboxId} for detection: ${detection.className}`); + const bboxId = await dataClient.addBoundingBoxToImageById({ + binaryId: "", + label: detection.className, + xMinNormalized: detection.boundingBox.xMin, + yMinNormalized: detection.boundingBox.yMin, + xMaxNormalized: detection.boundingBox.xMax, + yMaxNormalized: detection.boundingBox.yMax, + }); + + console.log( + `Added bounding box ID: ${bboxId} for detection: ${detection.className}`, + ); } ``` @@ -456,4 +452,4 @@ for (final detection in detections) { {{% /tab %}} {{< /tabs >}} -Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. \ No newline at end of file +Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 1a2c4d76e4..821954138c 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -109,22 +109,22 @@ fmt.Println("Image added to dataset successfully") ```typescript // Connect to Viam client const client: ViamClient = await createViamClient({ - credential: { - type: 'api-key', - payload: API_KEY, - }, - authEntity: API_KEY_ID, + credential: { + type: "api-key", + payload: API_KEY, + }, + authEntity: API_KEY_ID, }); const dataClient = client.dataClient; // Add image to dataset await dataClient.addBinaryDataToDatasetByIds({ - binaryIds: [EXISTING_IMAGE_ID], - datasetId: EXISTING_DATASET_ID, + binaryIds: [EXISTING_IMAGE_ID], + datasetId: EXISTING_DATASET_ID, }); -console.log('Image added to dataset successfully'); +console.log("Image added to dataset successfully"); client.disconnect(); ``` @@ -183,6 +183,7 @@ Re-train your ML model on the improved dataset to improve the ML model. {{< tabs >}} {{% tab name="Python" %}} + ```python import asyncio import os @@ -303,6 +304,7 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) ``` + {{% /tab %}} {{% tab name="Go" %}} @@ -376,7 +378,7 @@ func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImag return fmt.Errorf("failed to add image to dataset: %w", err) } - dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", + dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", classification, dc.datasetID, fileID, MachinePartID) return nil @@ -426,15 +428,16 @@ func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, }, nil } ``` + {{% /tab %}} {{% tab name="Typescript" %}} ```typescript -import { createRobotClient, RobotClient } from '@viamrobotics/sdk'; -import { AppClient, DataClient } from '@viamrobotics/app-client'; -import { Logger } from '@viamrobotics/utils'; +import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; +import { AppClient, DataClient } from "@viamrobotics/app-client"; +import { Logger } from "@viamrobotics/utils"; -const MACHINE_PART_ID: string = 'your-machine-part-id-here'; +const MACHINE_PART_ID: string = "your-machine-part-id-here"; interface FileUploadOptions { componentType?: string; @@ -455,7 +458,7 @@ export class DataCollector { componentName: string, datasetId: string, apiKeyId: string, - apiKey: string + apiKey: string, ) { this.logger = logger; this.componentName = componentName; @@ -466,10 +469,10 @@ export class DataCollector { async captureAndStoreImage( processedImage: ArrayBuffer, - classification: string + classification: string, ): Promise { if (!MACHINE_PART_ID) { - throw new Error('Machine part ID not configured'); + throw new Error("Machine part ID not configured"); } let dataClient: DataClient | null = null; @@ -481,8 +484,8 @@ export class DataCollector { const timestamp = Math.floor(Date.now() / 1000); const filename = `${classification}-sample-${timestamp}.png`; - const componentType = 'rdk:component:camera'; - const fileExtension = 'png'; + const componentType = "rdk:component:camera"; + const fileExtension = "png"; const uploadOptions: FileUploadOptions = { componentType, @@ -494,19 +497,15 @@ export class DataCollector { const fileId = await dataClient.fileUploadFromBytes( MACHINE_PART_ID, new Uint8Array(imageData), - uploadOptions + uploadOptions, ); - await dataClient.addBinaryDataToDatasetByIds( - [fileId], - this.datasetId - ); + await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); this.logger.info( `Successfully added ${classification} image to dataset ${this.datasetId} ` + - `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})` + `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, ); - } catch (error) { this.logger.error(`File upload failed for ${classification}: ${error}`); throw new Error(`Failed to upload image: ${error}`); @@ -519,7 +518,7 @@ export class DataCollector { private async createDataClient(): Promise { if (!this.apiKeyId || !this.apiKey) { - throw new Error('API credentials not configured'); + throw new Error("API credentials not configured"); } const appClient = new AppClient({ @@ -546,19 +545,19 @@ export class DataCollector { componentName: string, datasetId: string, apiKeyId: string, - apiKey: string + apiKey: string, ): DataCollector { if (!logger) { - throw new Error('Logger is required'); + throw new Error("Logger is required"); } if (!componentName) { - throw new Error('Component name is required'); + throw new Error("Component name is required"); } if (!datasetId) { - throw new Error('Dataset ID is required'); + throw new Error("Dataset ID is required"); } if (!apiKeyId || !apiKey) { - throw new Error('API credentials are required'); + throw new Error("API credentials are required"); } return new DataCollector( @@ -566,7 +565,7 @@ export class DataCollector { componentName, datasetId, apiKeyId, - apiKey + apiKey, ); } } diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index e54c428bd8..ee3517df5f 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -112,15 +112,15 @@ const dataClient = client.dataClient; console.log("Creating dataset..."); try { - const datasetId = await dataClient.createDataset({ - name: "", - organizationId: "", - }); - console.log(`Created dataset: ${datasetId}`); + const datasetId = await dataClient.createDataset({ + name: "", + organizationId: "", + }); + console.log(`Created dataset: ${datasetId}`); } catch (error) { - console.log("Error creating dataset. It may already exist."); - console.log(`Exception: ${error}`); - process.exit(1); + console.log("Error creating dataset. It may already exist."); + console.log(`Exception: ${error}`); + process.exit(1); } ``` @@ -154,4 +154,4 @@ final datasetId = await dataClient.createDataset( {{% /tab %}} {{< /tabs >}} -Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. \ No newline at end of file +Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 15909e50ce..323043c240 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -222,13 +222,13 @@ func main() { {{% tab name="TypeScript" %}} ```typescript -import { createAppClient } from '@viamrobotics/app-client'; -import { promises as fs } from 'fs'; +import { createAppClient } from "@viamrobotics/app-client"; +import { promises as fs } from "fs"; -const MACHINE_PART_ID = 'your-machine-part-id-here'; -const DATASET_ID = 'your-dataset-id-here'; -const API_KEY_ID = 'your-api-key-id'; -const API_KEY = 'your-api-key'; +const MACHINE_PART_ID = "your-machine-part-id-here"; +const DATASET_ID = "your-dataset-id-here"; +const API_KEY_ID = "your-api-key-id"; +const API_KEY = "your-api-key"; async function addImageToDataset(): Promise { // Create app client @@ -242,7 +242,7 @@ async function addImageToDataset(): Promise { const dataClient = appClient.dataClient(); // Read image file - const imagePath = 'sample_image.jpg'; + const imagePath = "sample_image.jpg"; const imageBuffer = await fs.readFile(imagePath); const imageData = new Uint8Array(imageBuffer); @@ -250,10 +250,10 @@ async function addImageToDataset(): Promise { const uploadResponse = await dataClient.fileUploadFromBytes({ partId: MACHINE_PART_ID, data: imageData, - componentType: 'camera', - componentName: 'camera-1', + componentType: "camera", + componentName: "camera-1", fileName: `training_sample_${Date.now()}.jpg`, - fileExtension: 'jpg', + fileExtension: "jpg", }); const fileId = uploadResponse.fileId; @@ -266,7 +266,6 @@ async function addImageToDataset(): Promise { }); console.log(`Image added to dataset ${DATASET_ID} successfully`); - } catch (error) { console.error(`Operation failed: ${error}`); throw error; @@ -277,10 +276,9 @@ async function addImageToDataset(): Promise { // Execute function addImageToDataset().catch((error) => { - console.error('Failed to add image to dataset:', error); + console.error("Failed to add image to dataset:", error); process.exit(1); }); - ``` {{% /tab %}} @@ -362,7 +360,6 @@ void main() async { {{% /tab %}} {{< /tabs >}} - ## Add all images captured by a specific machine to a dataset The following script adds all images captured from a certain machine to a new dataset: @@ -461,6 +458,7 @@ async def main() -> int: if __name__ == "__main__": asyncio.run(main()) ``` + {{% /tab %}} {{% tab name="Go" %}} @@ -589,15 +587,15 @@ func main() { {{% tab name="Typescript" %}} ```typescript -import { ViamClient, createViamClient } from '@viamrobotics/sdk'; -import { DataServiceClient } from '@viamrobotics/sdk/dist/gen/app/data/v1/data_pb_service'; -import { - BinaryDataByFilterRequest, - CreateDatasetRequest, - AddBinaryDataToDatasetByIdsRequest, - Filter, - DataRequest -} from '@viamrobotics/sdk/dist/gen/app/data/v1/data_pb'; +import { ViamClient, createViamClient } from "@viamrobotics/sdk"; +import { DataServiceClient } from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb_service"; +import { + BinaryDataByFilterRequest, + CreateDatasetRequest, + AddBinaryDataToDatasetByIdsRequest, + Filter, + DataRequest, +} from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb"; // Configuration constants - replace with your actual values const DATASET_NAME = ""; // a unique, new name for the dataset you want to create @@ -608,96 +606,101 @@ const API_KEY_ID = ""; // API key ID, find or create in your organization settin const MAX_MATCHES = 500; async function connect(): Promise { - return await createViamClient({ - credential: { - type: 'api-key', - authEntity: API_KEY_ID, - payload: API_KEY, - }, - }); + return await createViamClient({ + credential: { + type: "api-key", + authEntity: API_KEY_ID, + payload: API_KEY, + }, + }); } -async function fetchBinaryDataIds(dataClient: DataServiceClient, partId: string): Promise { - const filter = new Filter(); - filter.setPartId(partId); +async function fetchBinaryDataIds( + dataClient: DataServiceClient, + partId: string, +): Promise { + const filter = new Filter(); + filter.setPartId(partId); - const allMatches: any[] = []; - let last = ""; + const allMatches: any[] = []; + let last = ""; - console.log("Getting data for part..."); + console.log("Getting data for part..."); - while (allMatches.length < MAX_MATCHES) { - console.log("Fetching more data..."); + while (allMatches.length < MAX_MATCHES) { + console.log("Fetching more data..."); - const dataRequest = new DataRequest(); - dataRequest.setFilter(filter); - dataRequest.setLimit(50); - dataRequest.setLast(last); - dataRequest.setIncludeBinaryData(false); + const dataRequest = new DataRequest(); + dataRequest.setFilter(filter); + dataRequest.setLimit(50); + dataRequest.setLast(last); + dataRequest.setIncludeBinaryData(false); - const request = new BinaryDataByFilterRequest(); - request.setDataRequest(dataRequest); + const request = new BinaryDataByFilterRequest(); + request.setDataRequest(dataRequest); - const response = await dataClient.binaryDataByFilter(request); - const data = response.getDataList(); + const response = await dataClient.binaryDataByFilter(request); + const data = response.getDataList(); - if (data.length === 0) { - break; - } - - allMatches.push(...data); - last = response.getLast(); + if (data.length === 0) { + break; } - return allMatches; + allMatches.push(...data); + last = response.getLast(); + } + + return allMatches; } async function main(): Promise { - const viamClient = await connect(); - const dataClient = viamClient.dataClient; - - const matchingData = await fetchBinaryDataIds(dataClient, PART_ID); + const viamClient = await connect(); + const dataClient = viamClient.dataClient; - console.log("Creating dataset..."); + const matchingData = await fetchBinaryDataIds(dataClient, PART_ID); - try { - const createRequest = new CreateDatasetRequest(); - createRequest.setName(DATASET_NAME); - createRequest.setOrganizationId(ORG_ID); + console.log("Creating dataset..."); - const datasetResponse = await dataClient.createDataset(createRequest); - const datasetId = datasetResponse.getId(); - console.log(`Created dataset: ${datasetId}`); + try { + const createRequest = new CreateDatasetRequest(); + createRequest.setName(DATASET_NAME); + createRequest.setOrganizationId(ORG_ID); - console.log("Adding data to dataset..."); + const datasetResponse = await dataClient.createDataset(createRequest); + const datasetId = datasetResponse.getId(); + console.log(`Created dataset: ${datasetId}`); - const binaryIds = matchingData.map(obj => obj.getMetadata()?.getId() || ""); + console.log("Adding data to dataset..."); - const addRequest = new AddBinaryDataToDatasetByIdsRequest(); - addRequest.setBinaryIdsList(binaryIds); - addRequest.setDatasetId(datasetId); + const binaryIds = matchingData.map( + (obj) => obj.getMetadata()?.getId() || "", + ); - await dataClient.addBinaryDataToDatasetByIds(addRequest); + const addRequest = new AddBinaryDataToDatasetByIdsRequest(); + addRequest.setBinaryIdsList(binaryIds); + addRequest.setDatasetId(datasetId); - console.log("Added files to dataset."); - console.log(`See dataset: https://app.viam.com/data/datasets?id=${datasetId}`); + await dataClient.addBinaryDataToDatasetByIds(addRequest); - viamClient.disconnect(); - return 0; + console.log("Added files to dataset."); + console.log( + `See dataset: https://app.viam.com/data/datasets?id=${datasetId}`, + ); - } catch (error) { - console.log("Error creating dataset. It may already exist."); - console.log("See: https://app.viam.com/data/datasets"); - console.log(`Exception: ${error}`); - viamClient.disconnect(); - return 1; - } + viamClient.disconnect(); + return 0; + } catch (error) { + console.log("Error creating dataset. It may already exist."); + console.log("See: https://app.viam.com/data/datasets"); + console.log(`Exception: ${error}`); + viamClient.disconnect(); + return 1; + } } if (require.main === module) { - main().then(code => process.exit(code)); + main().then((code) => process.exit(code)); } - ``` {{% /tab %}} From a67c83196828d5ff55c0be374120a6ae03dae71c Mon Sep 17 00:00:00 2001 From: nathan contino Date: Wed, 25 Jun 2025 17:10:43 -0400 Subject: [PATCH 03/29] Fix broken link --- docs/data-ai/train/update-dataset.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 323043c240..e004a9f13f 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -39,7 +39,7 @@ Follow the guide to configure a [webcam](/operate/reference/components/camera/we Use the Viam CLI to filter images by label and add the filtered images to a dataset: -1. First, [create a dataset](#create-a-dataset), if you haven't already. +1. First, [create a dataset](/data-ai/train/create-dataset/), if you haven't already. 1. If you just created a dataset, use the dataset ID output by the creation command. If your dataset already exists, run the following command to get a list of dataset names and corresponding IDs: From 24275fd643d6bcfca32f59496b0ed4f59757ccfb Mon Sep 17 00:00:00 2001 From: nathan contino Date: Wed, 25 Jun 2025 17:14:17 -0400 Subject: [PATCH 04/29] Fix missing path, elongate description --- docs/data-ai/train/capture-images.md | 2 +- docs/data-ai/train/upload-external-data.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 821954138c..545ebd343b 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -4,7 +4,7 @@ title: "Capture images for training" weight: 20 layout: "docs" type: "docs" -description: "Capture images to train a machine learning model." +description: "Capture images that you can use to train a machine learning model." --- ## Prerequisites diff --git a/docs/data-ai/train/upload-external-data.md b/docs/data-ai/train/upload-external-data.md index d25af1dc12..a11721438c 100644 --- a/docs/data-ai/train/upload-external-data.md +++ b/docs/data-ai/train/upload-external-data.md @@ -12,6 +12,7 @@ aliases: - /services/data/upload/ - /how-tos/upload-data/ - /data-ai/ai/advanced/upload-external-data/ + - /data-ai/ai/advanced/ date: "2024-12-04" description: "Upload data to Viam from your local computer or mobile device using the data client API, Viam CLI, or Viam mobile app." prev: "/data-ai/ai/act/" From 7ee9e5f9ff348e37dfa3d37551d60cd9919c1458 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Wed, 25 Jun 2025 17:39:17 -0400 Subject: [PATCH 05/29] Fix most markdownlint complaints --- docs/data-ai/train/annotate-images.md | 3 +- docs/data-ai/train/capture-images.md | 15 +++- docs/data-ai/train/create-dataset.md | 16 ++-- docs/data-ai/train/update-dataset.md | 122 +++++++++++++------------- 4 files changed, 82 insertions(+), 74 deletions(-) diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/annotate-images.md index 74a80fe01c..e3c8fb5d60 100644 --- a/docs/data-ai/train/annotate-images.md +++ b/docs/data-ai/train/annotate-images.md @@ -78,7 +78,8 @@ detections = result.detections tags = ["tag1", "tag2"] -my_filter = create_filter(component_name="camera-1", organization_ids=[""]) +my_filter = + create_filter(component_name="camera-1", organization_ids=[""]) binary_metadata, count, last = await data_client.binary_data_by_filter( filter=my_filter, limit=20, diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 545ebd343b..fd33256917 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -199,14 +199,20 @@ from viam.logging import getLogger MACHINE_PART_ID = "your-machine-part-id-here" class DataCollector: - def __init__(self, component_name: str, dataset_id: str, api_key_id: str, api_key: str): + def __init__(self, + component_name: str, dataset_id: str, + api_key_id: str, api_key: str): + self.logger = getLogger(__name__) self.component_name = component_name self.dataset_id = dataset_id self.api_key_id = api_key_id self.api_key = api_key - async def capture_and_store_image(self, processed_image: Image.Image, classification: str) -> None: + async def capture_and_store_image(self, + processed_image: Image.Image, + classification: str) -> None: + if not MACHINE_PART_ID: raise ValueError("machine part ID not configured") @@ -273,7 +279,10 @@ class DataCollector: img.save(buffer, format='PNG', optimize=True) return buffer.getvalue() -def create_data_collector(component_name: str, dataset_id: str, api_key_id: str, api_key: str) -> DataCollector: +def create_data_collector(component_name: str, + dataset_id: str, api_key_id: str, + api_key: str) -> DataCollector: + if not component_name: raise ValueError("component name is required") if not dataset_id: diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index ee3517df5f..5377e5710d 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -58,15 +58,15 @@ data_client = viam_client.data_client print("Creating dataset...") try: - dataset_id = await data_client.create_dataset( - name="", - organization_id="", - ) - print(f"Created dataset: {dataset_id}") + dataset_id = await data_client.create_dataset( + name="", + organization_id="", + ) + print(f"Created dataset: {dataset_id}") except Exception as e: - print("Error creating dataset. It may already exist.") - print(f"Exception: {e}") - return 1 + print("Error creating dataset. It may already exist.") + print(f"Exception: {e}") + return 1 ``` {{% /tab %}} diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index e004a9f13f..933f956601 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -120,7 +120,6 @@ async def add_image_to_dataset(): # Execute function if __name__ == "__main__": asyncio.run(add_image_to_dataset()) - ``` {{% /tab %}} @@ -376,84 +375,83 @@ from viam.app.viam_client import ViamClient from viam.utils import create_filter # Configuration constants – replace with your actual values -DATASET_NAME = "" # a unique, new name for the dataset you want to create -ORG_ID = "" # your organization ID, find in your organization settings -PART_ID = "" # ID of machine that captured target images, find in machine config -API_KEY = "" # API key, find or create in your organization settings -API_KEY_ID = "" # API key ID, find or create in your organization settings +DATASET_NAME = "" # a unique, new name for the dataset you want to create +ORG_ID = "" # your organization ID, find in your organization settings +PART_ID = "" # ID of machine that captured target images, find in machine config +API_KEY = "" # API key, find or create in your organization settings +API_KEY_ID = "" # API key ID, find or create in your organization settings # Adjust the maximum number of images to add to the dataset MAX_MATCHES = 500 async def connect() -> ViamClient: - """Establish a connection to the Viam client using API credentials.""" - dial_options = DialOptions( - credentials=Credentials( - type="api-key", - payload=API_KEY, - ), - auth_entity=API_KEY_ID, - ) - return await ViamClient.create_from_dial_options(dial_options) + """Establish a connection to the Viam client using API credentials.""" + dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, + ) + return await ViamClient.create_from_dial_options(dial_options) async def fetch_binary_data_ids(data_client, part_id: str) -> List[str]: - """Fetch binary data metadata and return a list of BinaryData objects.""" - data_filter = create_filter(part_id=part_id) - all_matches = [] - last: Optional[str] = None - - print("Getting data for part...") - - while len(all_matches) < MAX_MATCHES: - print("Fetching more data...") - data, _, last = await data_client.binary_data_by_filter( - data_filter, - limit=50, - last=last, - include_binary_data=False, - ) - if not data: - break - all_matches.extend(data) + """Fetch binary data metadata and return a list of BinaryData objects.""" + data_filter = create_filter(part_id=part_id) + all_matches = [] + last: Optional[str] = None + + print("Getting data for part...") + + while len(all_matches) < MAX_MATCHES: + print("Fetching more data...") + data, _, last = await data_client.binary_data_by_filter( + data_filter, + limit=50, + last=last, + include_binary_data=False, + ) + if not data: + break + all_matches.extend(data) - return all_matches + return all_matches async def main() -> int: - """Main execution function.""" - viam_client = await connect() - data_client = viam_client.data_client + """Main execution function.""" + viam_client = await connect() + data_client = viam_client.data_client - matching_data = await fetch_binary_data_ids(data_client, PART_ID) + matching_data = await fetch_binary_data_ids(data_client, PART_ID) - print("Creating dataset...") + print("Creating dataset...") - try: - dataset_id = await data_client.create_dataset( - name=DATASET_NAME, - organization_id=ORG_ID, - ) - print(f"Created dataset: {dataset_id}") - except Exception as e: - print("Error creating dataset. It may already exist.") - print("See: https://app.viam.com/data/datasets") - print(f"Exception: {e}") - return 1 - - print("Adding data to dataset...") - - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[obj.metadata.binary_data_id for obj in matching_data], - dataset_id=dataset_id - ) - - print("Added files to dataset.") - print(f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}") + try: + dataset_id = await data_client.create_dataset( + name=DATASET_NAME, + organization_id=ORG_ID, + ) + print(f"Created dataset: {dataset_id}") + except Exception as e: + print("Error creating dataset. It may already exist.") + print("See: https://app.viam.com/data/datasets") + print(f"Exception: {e}") + return 1 + + print("Adding data to dataset...") + + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[obj.metadata.binary_data_id for obj in matching_data], + dataset_id=dataset_id + ) - viam_client.close() - return 0 + print("Added files to dataset.") + print(f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}") + viam_client.close() + return 0 if __name__ == "__main__": asyncio.run(main()) From fc6cb9c99deb3513974487d034913a04b2aed488 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Wed, 25 Jun 2025 17:53:55 -0400 Subject: [PATCH 06/29] Fix more linter errors --- docs/data-ai/train/annotate-images.md | 7 ++-- docs/data-ai/train/capture-images.md | 23 +++++++----- docs/data-ai/train/create-dataset.md | 2 +- docs/data-ai/train/update-dataset.md | 54 ++++++++++++++++++++++++--- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/annotate-images.md index e3c8fb5d60..1032cb041d 100644 --- a/docs/data-ai/train/annotate-images.md +++ b/docs/data-ai/train/annotate-images.md @@ -10,7 +10,7 @@ description: "Annotate images to train a machine learning model." Use the interface on the [**DATA** page](https://app.viam.com/data/view) to annotate your images. When you label your dataset, include: -- images with and _without_ the categories you’re looking to identify +- images with and _without_ the categories you're looking to identify - a roughly equal number of images for each category - images from your production environment, including lighting and camera quality - examples from every angle and distance that you expect the model to handle @@ -78,8 +78,9 @@ detections = result.detections tags = ["tag1", "tag2"] -my_filter = - create_filter(component_name="camera-1", organization_ids=[""]) +my_filter = create_filter( + component_name="camera-1", organization_ids=[""] +) binary_metadata, count, last = await data_client.binary_data_by_filter( filter=my_filter, limit=20, diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index fd33256917..5084857872 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -200,8 +200,8 @@ MACHINE_PART_ID = "your-machine-part-id-here" class DataCollector: def __init__(self, - component_name: str, dataset_id: str, - api_key_id: str, api_key: str): + component_name: str, dataset_id: str, + api_key_id: str, api_key: str): self.logger = getLogger(__name__) self.component_name = component_name @@ -210,8 +210,8 @@ class DataCollector: self.api_key = api_key async def capture_and_store_image(self, - processed_image: Image.Image, - classification: str) -> None: + processed_image: Image.Image, + classification: str) -> None: if not MACHINE_PART_ID: raise ValueError("machine part ID not configured") @@ -247,8 +247,9 @@ class DataCollector: ) self.logger.info( - f"successfully added {classification} image to dataset {self.dataset_id} " - f"(file ID: {file_id}, machine: {MACHINE_PART_ID})" + f"successfully added {classification} image to dataset " + f"{self.dataset_id} (file ID: {file_id}, " + f"machine: {MACHINE_PART_ID})" ) except Exception as e: @@ -279,10 +280,12 @@ class DataCollector: img.save(buffer, format='PNG', optimize=True) return buffer.getvalue() -def create_data_collector(component_name: str, - dataset_id: str, api_key_id: str, - api_key: str) -> DataCollector: - +def create_data_collector( + component_name: str, + dataset_id: str, + api_key_id: str, + api_key: str + ) -> DataCollector: if not component_name: raise ValueError("component name is required") if not dataset_id: diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 5377e5710d..cf75f2e85f 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -66,7 +66,7 @@ try: except Exception as e: print("Error creating dataset. It may already exist.") print(f"Exception: {e}") - return 1 + raise ``` {{% /tab %}} diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 933f956601..607d258fb5 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -66,11 +66,11 @@ from PIL import Image from viam.app.app_client import AppClient from viam.app.data_client import DataClient -MACHINE_PART_ID = "your-machine-part-id-here" DATASET_ID = "your-dataset-id-here" API_KEY_ID = "your-api-key-id" API_KEY = "your-api-key" + async def add_image_to_dataset(): """Add image to Viam dataset using Python SDK""" @@ -117,9 +117,47 @@ async def add_image_to_dataset(): finally: app_client.close() -# Execute function + +async def main() -> int: + """Main execution function.""" + viam_client = await connect() + data_client = viam_client.data_client + + matching_data = await fetch_binary_data_ids(data_client, PART_ID) + + print("Creating dataset...") + + try: + dataset_id = await data_client.create_dataset( + name=DATASET_NAME, + organization_id=ORG_ID, + ) + print(f"Created dataset: {dataset_id}") + except Exception as e: + print("Error creating dataset. It may already exist.") + print("See: https://app.viam.com/data/datasets") + print(f"Exception: {e}") + return 1 + + print("Adding data to dataset...") + + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[ + obj.metadata.binary_data_id for obj in matching_data + ], + dataset_id=dataset_id + ) + + print("Added files to dataset.") + print( + f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}" + ) + + viam_client.close() + return 0 + if __name__ == "__main__": - asyncio.run(add_image_to_dataset()) + asyncio.run(main()) ``` {{% /tab %}} @@ -443,18 +481,22 @@ async def main() -> int: print("Adding data to dataset...") await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[obj.metadata.binary_data_id for obj in matching_data], + binary_ids=[ + obj.metadata.binary_data_id for obj in matching_data + ], dataset_id=dataset_id ) print("Added files to dataset.") - print(f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}") + print( + f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}" + ) viam_client.close() return 0 if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) ``` {{% /tab %}} From a4e38dc8aedadfe659108270b5cb4f85673fec0a Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 08:53:28 -0400 Subject: [PATCH 07/29] Apply suggestions from code review Co-authored-by: Jessamy Taylor <75634662+JessamyT@users.noreply.github.com> --- docs/data-ai/train/capture-images.md | 5 +++-- docs/data-ai/train/create-dataset.md | 8 +++++--- docs/data-ai/train/train-tflite.md | 10 +++++----- docs/data-ai/train/update-dataset.md | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 5084857872..50fb097b9b 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -163,7 +163,7 @@ Once you've captured enough images for training, you must [annotate](/data-ai/tr ## Capture images over time -To capture a large number of images for training an ML model, [Capture and sync image data](/data-ai/capture-data/capture-sync/) using the data management service with your camera. +To capture a large number of images for training an ML model, [capture and sync image data](/data-ai/capture-data/capture-sync/) using the data management service with your camera. Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. We recommend you tag the images first and then use the CLI to [add the tagged images to a dataset](/data-ai/train/create-dataset/#add-tagged-images-to-a-dataset). @@ -178,7 +178,8 @@ Once you've captured enough images for training, you must [annotate](/data-ai/tr ## Capture, annotate, and add images to a dataset -The following example demonstrates how you can capture an image, use an ML model to generate annotations, then add the image to a dataset. You can use this logic to expand and improve your datasets continuously over time as your application runs. +The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. +You can use this logic to expand and improve your datasets continuously over time as your application runs. Re-train your ML model on the improved dataset to improve the ML model. {{< tabs >}} diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index cf75f2e85f..4f90829610 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -1,6 +1,6 @@ --- -linkTitle: "Create training dataset" -title: "Create training dataset" +linkTitle: "Create a training dataset" +title: "Create a training dataset" weight: 10 layout: "docs" type: "docs" @@ -13,7 +13,9 @@ aliases: - /data-ai/ai/create-dataset/ --- -To create a dataset: +To train a machine learning model, you will need a dataset. + +You can create a dataset using the web UI, the CLI, or one of the SDKs: {{< tabs >}} {{% tab name="Web UI" %}} diff --git a/docs/data-ai/train/train-tflite.md b/docs/data-ai/train/train-tflite.md index 7deecfca13..63cc63708e 100644 --- a/docs/data-ai/train/train-tflite.md +++ b/docs/data-ai/train/train-tflite.md @@ -38,13 +38,13 @@ Follow this guide to use your image data to train an ML model, so that your mach {{% /expand %}} -{{% expand "a dataset that meets training requirements" %}} +{{% expand "A dataset that meets training requirements" %}} -To train a model, your dataset must meet the following criteria: +To train a model, your dataset must contain the following: -- at least 15 images -- at least 80% of the images have labels -- for each training label, at least 10 bounding boxes +- At least 15 images +- At least 80% of the images have labels +- For each training label, at least 10 bounding boxes Follow the guide to [create a dataset](/data-ai/train/create-dataset/). diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 607d258fb5..6450394063 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -32,7 +32,7 @@ Follow the guide to configure a [webcam](/operate/reference/components/camera/we 1. Use the checkbox in the upper left of each image to select labeled images. -1. Click the **Add to dataset** button, select a dataset, and click the **Add ... images** button to add the selected images to the dataset. +1. Click the **Add to dataset** button, select a dataset, and click the **Add `` images** button to add the selected images to the dataset. {{% /tab %}} {{% tab name="CLI" %}} From 6a5f3fe666997db97ec7a71578b38e6612856251 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 09:23:21 -0400 Subject: [PATCH 08/29] Implement additional docs feedback --- docs/data-ai/train/annotate-images.md | 100 ++++++++++++++------------ docs/data-ai/train/capture-images.md | 53 +++++++------- docs/data-ai/train/create-dataset.md | 18 +++-- docs/data-ai/train/update-dataset.md | 21 +++--- 4 files changed, 99 insertions(+), 93 deletions(-) diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/annotate-images.md index 1032cb041d..cce546b6fc 100644 --- a/docs/data-ai/train/annotate-images.md +++ b/docs/data-ai/train/annotate-images.md @@ -56,14 +56,11 @@ To tag an image: Repeat these steps for all images in the dataset. {{% /tab %}} -{{% tab name="SDK" %}} - -Use an ML model to generate tags for an image. -The following code shows how to add tags to an image in Viam: - -{{< tabs >}} {{% tab name="Python" %}} +Use an ML model to generate tags for an image or set of images. +Then, pass the tags and image IDs to [`data_client.add_tags_to_binary_data_by_ids`](/dev/reference/apis/data-client/#addtagstobinarydatabyids): + ```python detector = VisionClient.from_robot(machine, "") @@ -100,6 +97,9 @@ binary_data = await data_client.add_tags_to_binary_data_by_ids(tags, my_ids) {{% /tab %}} {{% tab name="Go" %}} +Use an ML model to generate tags for an image or set of images. +Then, pass the tags and image IDs to [`DataClient.AddTagsToBinaryDataByIDs`](/dev/reference/apis/data-client/#addtagstobinarydatabyids): + ```go ctx := context.Background() @@ -161,6 +161,9 @@ if err != nil { {{% /tab %}} {{% tab name="TypeScript" %}} +Use an ML model to generate tags for an image or set of images. +Then, pass the tags and image IDs to [`dataClient.addTagsToBinaryDataByIds`](/dev/reference/apis/data-client/#addtagstobinarydatabyids): + ```typescript const client = await createViamClient(); const myDetector = new VisionClient(client, ""); @@ -199,48 +202,48 @@ await dataClient.addTagsToBinaryDataByIds(tags, myIds); {{% /tab %}} {{% tab name="Flutter" %}} +Use an ML model to generate tags for an image or set of images. +Then, pass the tags and image IDs to [`dataClient.addTagsToBinaryDataByIds`](/dev/reference/apis/data-client/#addtagstobinarydatabyids): + ```dart - final viamClient = await ViamClient.connect(); - final myDetector = VisionClient.fromRobot(viamClient, ""); - final dataClient = viamClient.dataClient; +final viamClient = await ViamClient.connect(); +final myDetector = VisionClient.fromRobot(viamClient, ""); +final dataClient = viamClient.dataClient; - // Get the captured data for a camera - final result = await myDetector.captureAllFromCamera( +// Get the captured data for a camera +final result = await myDetector.captureAllFromCamera( "", returnImage: true, returnDetections: true, - ); - final image = result.image; - final detections = result.detections; +); +final image = result.image; +final detections = result.detections; - final tags = ["tag1", "tag2"]; +final tags = ["tag1", "tag2"]; - final myFilter = createFilter( +final myFilter = createFilter( componentName: "camera-1", organizationIds: [""], - ); +); - final binaryResult = await dataClient.binaryDataByFilter( +final binaryResult = await dataClient.binaryDataByFilter( filter: myFilter, limit: 20, includeBinaryData: false, - ); +); - final myIds = []; +final myIds = []; - for (final obj in binaryResult.binaryMetadata) { +for (final obj in binaryResult.binaryMetadata) { myIds.add(obj.metadata.binaryDataId); - } +} - await dataClient.addTagsToBinaryDataByIds(tags, myIds); +await dataClient.addTagsToBinaryDataByIds(tags, myIds); ``` {{% /tab %}} {{< /tabs >}} -{{% /tab %}} -{{< /tabs >}} - Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. ## Detect objects with bounding boxes @@ -273,13 +276,10 @@ Once created, you can move, resize, or delete the bounding box. Repeat these steps for all images in the dataset. {{% /tab %}} -{{% tab name="SDK" %}} +{{% tab name="Python" %}} Use an ML model to generate bounding boxes for an image. -The following code shows how to add bounding boxes to an image in Viam: - -{{< tabs >}} -{{% tab name="Python" %}} +Then, separately pass each bounding box and the image ID to [`data_client.add_bounding_box_to_image_by_id`](/dev/reference/apis/data-client/#addboundingboxtoimagebyid): ```python detector = VisionClient.from_robot(machine, "") @@ -341,6 +341,9 @@ for detection in detections: {{% /tab %}} {{% tab name="Go" %}} +Use an ML model to generate bounding boxes for an image. +Then, separately pass each bounding box and the image ID to [`DataClient.AddBoundingBoxToImageByID`](/dev/reference/apis/data-client/#addboundingboxtoimagebyid): + ```go ctx := context.Background() @@ -386,6 +389,9 @@ for _, detection := range detections { {{% /tab %}} {{% tab name="TypeScript" %}} +Use an ML model to generate bounding boxes for an image. +Then, separately pass each bounding box and the image ID to [`dataClient.addBoundingBoxToImageById`](/dev/reference/apis/data-client/#addboundingboxtoimagebyid): + ```typescript const client = await createViamClient(); const myDetector = new VisionClient(client, ""); @@ -419,6 +425,9 @@ for (const detection of detections) { {{% /tab %}} {{% tab name="Flutter" %}} +Use an ML model to generate bounding boxes for an image. +Then, separately pass each bounding box and the image ID to [`dataClient.addBoundingBoxToImageById`](/dev/reference/apis/data-client/#addboundingboxtoimagebyid): + ```dart final viamClient = await ViamClient.connect(); final myDetector = VisionClient.fromRobot(viamClient, ""); @@ -426,32 +435,29 @@ final dataClient = viamClient.dataClient; // Get the captured data for a camera final result = await myDetector.captureAllFromCamera( - "", - returnImage: true, - returnDetections: true, + "", + returnImage: true, + returnDetections: true, ); final image = result.image; final detections = result.detections; // Process each detection and add bounding boxes for (final detection in detections) { - final bboxId = await dataClient.addBoundingBoxToImageById( - binaryId: "", - label: detection.className, - xMinNormalized: detection.boundingBox.xMin, - yMinNormalized: detection.boundingBox.yMin, - xMaxNormalized: detection.boundingBox.xMax, - yMaxNormalized: detection.boundingBox.yMax, - ); - - print('Added bounding box ID: $bboxId for detection: ${detection.className}'); + final bboxId = await dataClient.addBoundingBoxToImageById( + binaryId: "", + label: detection.className, + xMinNormalized: detection.boundingBox.xMin, + yMinNormalized: detection.boundingBox.yMin, + xMaxNormalized: detection.boundingBox.xMax, + yMaxNormalized: detection.boundingBox.yMax, + ); + + print('Added bounding box ID: $bboxId for detection: ${detection.className}'); } ``` {{% /tab %}} {{< /tabs >}} -{{% /tab %}} -{{< /tabs >}} - Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 50fb097b9b..9c57c72d35 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -40,13 +40,10 @@ To add an image directly to a dataset from a visual feed, complete the following To view images added to your dataset, go to the **DATA** page, open the [**DATASETS** tab](https://app.viam.com/data/datasets), then select your dataset. {{% /tab %}} -{{% tab name="SDK" %}} +{{% tab name="Python" %}} To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Each SDK provides a method to add an image to a dataset using those IDs: - -{{< tabs >}} -{{% tab name="Python" %}} +Pass both IDs to [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): ```python # Connect to Viam client @@ -74,24 +71,21 @@ viam_client.close() {{% /tab %}} {{% tab name="Go" %}} +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`DataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + ```go -// Connect to Viam client -client, err := app.NewViamClient( - context.Background() -) +ctx := context.Background() +viamClient, err := client.New(ctx, "", logger) if err != nil { - return fmt.Errorf("failed to create client: %w", err) + log.Fatal(err) } -defer client.Close() +defer viamClient.Close(ctx) -// Authenticate -err = client.LoginWithAPIKey(context.Background(), APIKeyID, APIKey) -if err != nil { - return fmt.Errorf("failed to authenticate: %w", err) -} +dataClient := viamClient.DataClient() // Add image to dataset -err = client.AddBinaryDataToDatasetByIDs( +err = dataClient.AddBinaryDataToDatasetByIDs( context.Background(), []string{ExistingImageID}, ExistingDatasetID, @@ -106,7 +100,11 @@ fmt.Println("Image added to dataset successfully") {{% /tab %}} {{% tab name="TypeScript" %}} +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + ```typescript + // Connect to Viam client const client: ViamClient = await createViamClient({ credential: { @@ -131,16 +129,20 @@ client.disconnect(); {{% /tab %}} {{% tab name="Flutter" %}} + +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + ```dart - // Connect to Viam client - final client = await ViamClient.withApiKey( +// Connect to Viam client +final client = await ViamClient.withApiKey( apiKeyId: apiKeyId, apiKey: apiKey, - ); +); - final dataClient = client.dataClient; +final dataClient = client.dataClient; - try { +try { // Add image to dataset await dataClient.addBinaryDataToDatasetByIds( binaryIds: [existingImageId], @@ -148,17 +150,14 @@ client.disconnect(); ); print('Image added to dataset successfully'); - } finally { +} finally { await client.close(); - } +} ``` {{% /tab %}} {{< /tabs >}} -{{% /tab %}} -{{< /tabs >}} - Once you've captured enough images for training, you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. ## Capture images over time diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 4f90829610..18cc69e9f2 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -17,8 +17,7 @@ To train a machine learning model, you will need a dataset. You can create a dataset using the web UI, the CLI, or one of the SDKs: -{{< tabs >}} -{{% tab name="Web UI" %}} +## Web UI 1. Navigate to the **DATA** page and open the [**DATASETS** tab](https://app.viam.com/data/datasets). @@ -30,8 +29,9 @@ You can create a dataset using the web UI, the CLI, or one of the SDKs: 1. Click **Create dataset** to create the dataset. -{{% /tab %}} -{{% tab name="CLI" %}} +Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. + +## CLI 1. First, install the Viam CLI and authenticate: @@ -45,8 +45,9 @@ You can create a dataset using the web UI, the CLI, or one of the SDKs: viam dataset create --org-id= --name= ``` -{{% /tab %}} -{{% tab name="SDK" %}} +Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. + +## SDK {{< tabs >}} {{% tab name="Python" %}} @@ -74,7 +75,7 @@ except Exception as e: {{% /tab %}} {{% tab name="Go" %}} -To create a dataset, pass a unique dataset name and organization ID to `dataClient.CreateDataset`: +To create a dataset, pass a unique dataset name and organization ID to `DataClient.CreateDataset`: ```go ctx := context.Background() @@ -153,7 +154,4 @@ final datasetId = await dataClient.createDataset( {{% /tab %}} {{< /tabs >}} -{{% /tab %}} -{{< /tabs >}} - Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 6450394063..a28f2a3919 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -32,7 +32,7 @@ Follow the guide to configure a [webcam](/operate/reference/components/camera/we 1. Use the checkbox in the upper left of each image to select labeled images. -1. Click the **Add to dataset** button, select a dataset, and click the **Add `` images** button to add the selected images to the dataset. +1. Click the **Add to dataset** button, select a dataset, and click the **Add \ images** button to add the selected images to the dataset. {{% /tab %}} {{% tab name="CLI" %}} @@ -55,11 +55,10 @@ Use the Viam CLI to filter images by label and add the filtered images to a data ``` {{% /tab %}} -{{% tab name="SDK" %}} - -{{< tabs >}} {{% tab name="Python" %}} +To add an image to a dataset, use [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + ```python import asyncio from PIL import Image @@ -163,6 +162,8 @@ if __name__ == "__main__": {{% /tab %}} {{% tab name="Go" %}} +To add an image to a dataset, use [`dataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + ```go package main @@ -258,6 +259,9 @@ func main() { {{% /tab %}} {{% tab name="TypeScript" %}} +To add an image to a dataset, use [`dataClient.addBinaryDataToDatasetByIds`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + + ```typescript import { createAppClient } from "@viamrobotics/app-client"; import { promises as fs } from "fs"; @@ -321,6 +325,8 @@ addImageToDataset().catch((error) => { {{% /tab %}} {{% tab name="Flutter" %}} +To add an image to a dataset, use [`dataClient.addBinaryDataToDatasetByIds`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + ```dart import 'dart:io'; import 'dart:typed_data'; @@ -394,15 +400,12 @@ void main() async { {{% /tab %}} {{< /tabs >}} -{{% /tab %}} -{{< /tabs >}} - ## Add all images captured by a specific machine to a dataset The following script adds all images captured from a certain machine to a new dataset: {{< tabs >}} -{{% tab name="Python" %}} +{{% tab name="Python SDK" %}} ```python import asyncio @@ -500,7 +503,7 @@ if __name__ == "__main__": ``` {{% /tab %}} -{{% tab name="Go" %}} +{{% tab name="Go SDK" %}} ```go package main From 9fdd043b922e6ee756aba218e97a450718f1a603 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 09:24:35 -0400 Subject: [PATCH 09/29] add more links --- docs/data-ai/train/create-dataset.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 18cc69e9f2..91de84e60b 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -52,7 +52,7 @@ Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [ad {{< tabs >}} {{% tab name="Python" %}} -To create a dataset, pass a unique dataset name and organization ID to `data_client.create_dataset`: +To create a dataset, pass a unique dataset name and organization ID to [`data_client.create_dataset`](/dev/reference/apis/data-client/#createdataset): ```python viam_client = await connect() @@ -75,7 +75,7 @@ except Exception as e: {{% /tab %}} {{% tab name="Go" %}} -To create a dataset, pass a unique dataset name and organization ID to `DataClient.CreateDataset`: +To create a dataset, pass a unique dataset name and organization ID to [`DataClient.CreateDataset`](/dev/reference/apis/data-client/#createdataset): ```go ctx := context.Background() @@ -106,7 +106,7 @@ fmt.Printf("Created dataset: %s\n", datasetID) {{% /tab %}} {{% tab name="TypeScript" %}} -To create a dataset, pass a unique dataset name and organization ID to `dataClient.createDataset`: +To create a dataset, pass a unique dataset name and organization ID to [`dataClient.createDataset`](/dev/reference/apis/data-client/#createdataset): ```typescript const client = await createViamClient(); @@ -130,7 +130,7 @@ try { {{% /tab %}} {{% tab name="Flutter" %}} -To create a dataset, pass a unique dataset name and organization ID to `dataClient.createDataset`: +To create a dataset, pass a unique dataset name and organization ID to [`dataClient.createDataset`](/dev/reference/apis/data-client/#createdataset): ```dart final viamClient = await ViamClient.connect(); From 2cfdde6ff18ae8551e984ac8d77d1bf8fd4316a0 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 12:25:44 -0400 Subject: [PATCH 10/29] Fix lint errors, update capture examples --- docs/data-ai/train/capture-images.md | 744 +++++----------------- docs/data-ai/train/create-dataset.md | 8 +- docs/data-ai/train/train-tflite.md | 2 +- docs/data-ai/train/update-dataset.md | 916 ++++++++++++++++++--------- 4 files changed, 777 insertions(+), 893 deletions(-) diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 9c57c72d35..5db329febe 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -21,6 +21,12 @@ Follow the guide to configure a [webcam](/operate/reference/components/camera/we {{% /expand%}} +{{< alert title="Tip" color="tip" >}} + +For the best results, use the same camera for both training data capture and production deployment. + +{{< /alert >}} + ## Capture individual images {{< tabs >}} @@ -42,11 +48,13 @@ To view images added to your dataset, go to the **DATA** page, open the [**DATAS {{% /tab %}} {{% tab name="Python" %}} -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`data_client.binary_data_capture_upload`](/dev/reference/apis/data-client/#binarydatacaptureupload): ```python -# Connect to Viam client +CAMERA_NAME = "" +MACHINE_ADDRESS = "" + dial_options = DialOptions( credentials=Credentials( type="api-key", @@ -55,28 +63,70 @@ dial_options = DialOptions( auth_entity=API_KEY_ID, ) +robot_opts = RobotClient.Options.with_api_key( + api_key=API_KEY, + api_key_id=API_KEY_ID +) + viam_client = await ViamClient.create_from_dial_options(dial_options) data_client = viam_client.data_client -# Add image to dataset -await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[EXISTING_IMAGE_ID], - dataset_id=EXISTING_DATASET_ID +robot_client = await RobotClient.at_address(ROBOT_ADDRESS, robot_opts) +camera = Camera.from_robot(robot_client, CAMERA_NAME) + +# Capture image +image_frame = await camera.get_image() + +# Upload data +file_id = await data_client.binary_data_capture_upload( + part_id=PART_ID, + component_type="camera", + component_name=CAMERA_NAME, + method_name="GetImages", + data_request_times=[datetime.utcnow(), datetime.utcnow()], + file_extension=".jpg", + binary_data=image_frame ) -print("Image added to dataset successfully") +# Cleanup +await robot_client.close() viam_client.close() ``` {{% /tab %}} {{% tab name="Go" %}} -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`DataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`DataClient.BinaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): ```go +const ( + CAMERA_NAME = "" + MACHINE_ADDRESS = "" + API_KEY = "" + API_KEY_ID = "" + PART_ID = "" +) + ctx := context.Background() -viamClient, err := client.New(ctx, "", logger) +machine, err := client.New( + ctx, + MACHINE_ADDRESS, + logger, + client.WithDialOptions(rpc.WithEntityCredentials( + API_KEY_ID, + rpc.Credentials{ + Type: rpc.CredentialsTypeAPIKey, + Payload: API_KEY, + }, + )), +) +if err != nil { + return "", err +} +defer machine.Close(ctx) + +viamClient, err := client.New(ctx, MACHINE_ADDRESS, logger) if err != nil { log.Fatal(err) } @@ -84,28 +134,52 @@ defer viamClient.Close(ctx) dataClient := viamClient.DataClient() -// Add image to dataset -err = dataClient.AddBinaryDataToDatasetByIDs( - context.Background(), - []string{ExistingImageID}, - ExistingDatasetID, -) +camera, err := camera.FromRobot(machine, CAMERA_NAME) if err != nil { - return fmt.Errorf("failed to add image to dataset: %w", err) + return "", err } -fmt.Println("Image added to dataset successfully") +// Capture image +img, _, err := camera.GetImage(ctx) +if err != nil { + return "", err +} + +// Upload binary data +now := time.Now().UTC() +fileID, err := dataClient.BinaryDataCaptureUpload(ctx, app.BinaryDataCaptureUploadOptions{ + PartID: PART_ID, + ComponentType: "camera", + ComponentName: CAMERA_NAME, + MethodName: "GetImages", + DataRequestTimes: []time.Time{now, now}, + FileExtension: ".jpg", + BinaryData: img, +}) ``` {{% /tab %}} {{% tab name="TypeScript" %}} -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): ```typescript +const CAMERA_NAME = ''; +const MACHINE_ADDRESS = ''; +const API_KEY = ''; +const API_KEY_ID = ''; +const PART_ID = ''; + +const machine = await Viam.createRobotClient({ + host: MACHINE_ADDRESS, + credential: { + type: 'api-key', + payload: API_KEY, + }, + authEntity: API_KEY_ID, +}); -// Connect to Viam client const client: ViamClient = await createViamClient({ credential: { type: "api-key", @@ -116,56 +190,89 @@ const client: ViamClient = await createViamClient({ const dataClient = client.dataClient; -// Add image to dataset -await dataClient.addBinaryDataToDatasetByIds({ - binaryIds: [EXISTING_IMAGE_ID], - datasetId: EXISTING_DATASET_ID, +const camera = new Viam.CameraClient(machine, CAMERA_NAME); + +// Capture image +const imageFrame = await camera.getImage(); + +// Upload binary data +const now = new Date(); +const fileId = await dataClient.binaryDataCaptureUpload({ + partId: PART_ID, + componentType: 'camera', + componentName: CAMERA_NAME, + methodName: 'GetImages', + dataRequestTimes: [now, now], + fileExtension: '.jpg', + binaryData: imageFrame, }); -console.log("Image added to dataset successfully"); -client.disconnect(); +// Cleanup +await machine.disconnect(); +dataClient.close(); ``` {{% /tab %}} {{% tab name="Flutter" %}} - -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): ```dart -// Connect to Viam client +const String CAMERA_NAME = ''; +const String MACHINE_ADDRESS = ''; +const String API_KEY = ''; +const String API_KEY_ID = ''; +const String PART_ID = ''; + +final machine = await RobotClient.atAddress( + MACHINE_ADDRESS, + RobotClientOptions.withApiKey( + apiKey: API_KEY, + apiKeyId: API_KEY_ID, + ), +); + final client = await ViamClient.withApiKey( - apiKeyId: apiKeyId, - apiKey: apiKey, + apiKeyId: API_KEY_ID, + apiKey: API_KEY, ); final dataClient = client.dataClient; -try { - // Add image to dataset - await dataClient.addBinaryDataToDatasetByIds( - binaryIds: [existingImageId], - datasetId: existingDatasetId, - ); +final camera = Camera.fromRobot(machine, CAMERA_NAME); + +// Capture image +final imageFrame = await camera.getImage(); + +// Upload binary data +final now = DateTime.now().toUtc(); +final fileId = await dataClient.binaryDataCaptureUpload( + partId: PART_ID, + componentType: 'camera', + componentName: CAMERA_NAME, + methodName: 'GetImages', + dataRequestTimes: [now, now], + fileExtension: '.jpg', + binaryData: imageFrame, +); - print('Image added to dataset successfully'); -} finally { - await client.close(); -} +// Cleanup +await robotClient.close(); +dataClient.close(); ``` {{% /tab %}} {{< /tabs >}} -Once you've captured enough images for training, you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. +Once you've captured [enough images for training](/data-ai/train/train-tflite/), you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. ## Capture images over time -To capture a large number of images for training an ML model, [capture and sync image data](/data-ai/capture-data/capture-sync/) using the data management service with your camera. +To capture a large number of images for training an ML model, use the data management service to [capture and sync image data](/data-ai/capture-data/capture-sync/) from your camera. -Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. -We recommend you tag the images first and then use the CLI to [add the tagged images to a dataset](/data-ai/train/create-dataset/#add-tagged-images-to-a-dataset). +When you sync with data management, Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. +To use your captured images for training, [annotate the images](data-ai/train/annotate-images/), then [add the images to a dataset](/data-ai/train/update-dataset/). {{< alert title="Tip" color="tip" >}} @@ -173,543 +280,4 @@ Once you have enough images, consider disabling data capture to [avoid incurring {{< /alert >}} -Once you've captured enough images for training, you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. - -## Capture, annotate, and add images to a dataset - -The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. -You can use this logic to expand and improve your datasets continuously over time as your application runs. -Re-train your ML model on the improved dataset to improve the ML model. - -{{< tabs >}} -{{% tab name="Python" %}} - -```python -import asyncio -import os -import time -from typing import Optional -from io import BytesIO - -from PIL import Image -from viam.app.app_client import AppClient -from viam.logging import getLogger - -# Global machine configuration -MACHINE_PART_ID = "your-machine-part-id-here" - -class DataCollector: - def __init__(self, - component_name: str, dataset_id: str, - api_key_id: str, api_key: str): - - self.logger = getLogger(__name__) - self.component_name = component_name - self.dataset_id = dataset_id - self.api_key_id = api_key_id - self.api_key = api_key - - async def capture_and_store_image(self, - processed_image: Image.Image, - classification: str) -> None: - - if not MACHINE_PART_ID: - raise ValueError("machine part ID not configured") - - # Create fresh data client connection - async with await self._create_data_client() as data_client: - image_data = self._encode_image_to_png(processed_image) - - # Generate unique filename with timestamp - timestamp = int(time.time()) - filename = f"{classification}-sample-{timestamp}.png" - - component_type = "rdk:component:camera" - - upload_metadata = { - "component_type": component_type, - "component_name": self.component_name, - "file_name": filename, - "file_extension": "png" - } - - try: - file_id = await data_client.file_upload_from_bytes( - part_id=MACHINE_PART_ID, - data=image_data, - **upload_metadata - ) - - # Associate file with dataset immediately - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[file_id], - dataset_id=self.dataset_id - ) - - self.logger.info( - f"successfully added {classification} image to dataset " - f"{self.dataset_id} (file ID: {file_id}, " - f"machine: {MACHINE_PART_ID})" - ) - - except Exception as e: - self.logger.error(f"failed to upload and associate image: {e}") - raise - - async def _create_data_client(self) -> AppClient: - if not self.api_key_id or not self.api_key: - raise ValueError("API credentials not configured") - - try: - client = AppClient.create_from_dial_options( - dial_options={ - "auth_entity": self.api_key_id, - "credentials": { - "type": "api-key", - "payload": self.api_key - } - } - ) - return client - - except Exception as e: - raise ValueError(f"failed to create app client: {e}") - - def _encode_image_to_png(self, img: Image.Image) -> bytes: - buffer = BytesIO() - img.save(buffer, format='PNG', optimize=True) - return buffer.getvalue() - -def create_data_collector( - component_name: str, - dataset_id: str, - api_key_id: str, - api_key: str - ) -> DataCollector: - if not component_name: - raise ValueError("component name is required") - if not dataset_id: - raise ValueError("dataset ID is required") - if not api_key_id or not api_key: - raise ValueError("API credentials are required") - - return DataCollector( - component_name=component_name, - dataset_id=dataset_id, - api_key_id=api_key_id, - api_key=api_key - ) - -# Example usage -async def main(): - collector = create_data_collector( - component_name="main_camera", - dataset_id="your-dataset-id", - api_key_id="your-api-key-id", - api_key="your-api-key" - ) - - # Example with PIL Image - sample_image = Image.new('RGB', (640, 480), color='red') - await collector.capture_and_store_image(sample_image, "positive_sample") - -if __name__ == "__main__": - asyncio.run(main()) -``` - -{{% /tab %}} -{{% tab name="Go" %}} - -```go -package datasetcreation - -import ( - "context" - "fmt" - "image" - "time" - - "go.viam.com/rdk/logging" - "go.viam.com/rdk/services/datamanager/app" - "go.viam.com/rdk/app/appclient" -) - -var MachinePartID string = "your-machine-part-id-here" - -type DataCollector struct { - logger logging.Logger - componentName string - datasetID string - apiKeyID string - apiKey string -} - -func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { - if MachinePartID == "" { - return fmt.Errorf("machine part ID not configured") - } - - // Create fresh data client connection - dataClient, err := dc.createDataClient(ctx) - if err != nil { - dc.logger.Errorf("failed to create data client: %v", err) - return fmt.Errorf("data client initialization failed: %w", err) - } - defer dataClient.Close() - - imageData, err := dc.encodeImageToPNG(processedImage) - if err != nil { - dc.logger.Errorf("image encoding failed: %v", err) - return fmt.Errorf("failed to encode image: %w", err) - } - - // Generate unique filename with timestamp - timestamp := time.Now().Unix() - filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) - - componentType := "rdk:component:camera" - fileExtension := "png" - - uploadOptions := app.FileUploadOptions{ - ComponentType: &componentType, - ComponentName: &dc.componentName, - FileName: &filename, - FileExtension: &fileExtension, - } - - fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) - if err != nil { - dc.logger.Errorf("file upload failed for %s: %v", filename, err) - return fmt.Errorf("failed to upload image: %w", err) - } - - // Associate file with dataset immediately - err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) - if err != nil { - dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) - return fmt.Errorf("failed to add image to dataset: %w", err) - } - - dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", - classification, dc.datasetID, fileID, MachinePartID) - - return nil -} - -func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { - if dc.apiKeyID == "" || dc.apiKey == "" { - return nil, fmt.Errorf("API credentials not configured") - } - - client, err := appclient.New(ctx, appclient.Config{ - KeyID: dc.apiKeyID, - Key: dc.apiKey, - }) - if err != nil { - return nil, fmt.Errorf("failed to create app client: %w", err) - } - - return client.DataClient, nil -} - -func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { - // PNG encoding implementation - return nil, nil // Placeholder -} - -func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { - if logger == nil { - return nil, fmt.Errorf("logger is required") - } - if componentName == "" { - return nil, fmt.Errorf("component name is required") - } - if datasetID == "" { - return nil, fmt.Errorf("dataset ID is required") - } - if apiKeyID == "" || apiKey == "" { - return nil, fmt.Errorf("API credentials are required") - } - - return &DataCollector{ - logger: logger, - componentName: componentName, - datasetID: datasetID, - apiKeyID: apiKeyID, - apiKey: apiKey, - }, nil -} -``` - -{{% /tab %}} -{{% tab name="Typescript" %}} - -```typescript -import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; -import { AppClient, DataClient } from "@viamrobotics/app-client"; -import { Logger } from "@viamrobotics/utils"; - -const MACHINE_PART_ID: string = "your-machine-part-id-here"; - -interface FileUploadOptions { - componentType?: string; - componentName?: string; - fileName?: string; - fileExtension?: string; -} - -export class DataCollector { - private logger: Logger; - private componentName: string; - private datasetId: string; - private apiKeyId: string; - private apiKey: string; - - constructor( - logger: Logger, - componentName: string, - datasetId: string, - apiKeyId: string, - apiKey: string, - ) { - this.logger = logger; - this.componentName = componentName; - this.datasetId = datasetId; - this.apiKeyId = apiKeyId; - this.apiKey = apiKey; - } - - async captureAndStoreImage( - processedImage: ArrayBuffer, - classification: string, - ): Promise { - if (!MACHINE_PART_ID) { - throw new Error("Machine part ID not configured"); - } - - let dataClient: DataClient | null = null; - - try { - dataClient = await this.createDataClient(); - - const imageData = await this.encodeImageToPng(processedImage); - const timestamp = Math.floor(Date.now() / 1000); - const filename = `${classification}-sample-${timestamp}.png`; - - const componentType = "rdk:component:camera"; - const fileExtension = "png"; - - const uploadOptions: FileUploadOptions = { - componentType, - componentName: this.componentName, - fileName: filename, - fileExtension, - }; - - const fileId = await dataClient.fileUploadFromBytes( - MACHINE_PART_ID, - new Uint8Array(imageData), - uploadOptions, - ); - - await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); - - this.logger.info( - `Successfully added ${classification} image to dataset ${this.datasetId} ` + - `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, - ); - } catch (error) { - this.logger.error(`File upload failed for ${classification}: ${error}`); - throw new Error(`Failed to upload image: ${error}`); - } finally { - if (dataClient) { - await dataClient.close(); - } - } - } - - private async createDataClient(): Promise { - if (!this.apiKeyId || !this.apiKey) { - throw new Error("API credentials not configured"); - } - - const appClient = new AppClient({ - apiKeyId: this.apiKeyId, - apiKey: this.apiKey, - }); - - return appClient.dataClient(); - } - - private async encodeImageToPng(image: ArrayBuffer): Promise { - try { - // PNG encoding implementation would depend on your image processing library - // This is a placeholder - you would use a library like 'pngjs' or 'canvas' - return image; // Assuming image is already PNG encoded - } catch (error) { - this.logger.error(`Image encoding failed: ${error}`); - throw new Error(`Failed to encode image: ${error}`); - } - } - - static create( - logger: Logger, - componentName: string, - datasetId: string, - apiKeyId: string, - apiKey: string, - ): DataCollector { - if (!logger) { - throw new Error("Logger is required"); - } - if (!componentName) { - throw new Error("Component name is required"); - } - if (!datasetId) { - throw new Error("Dataset ID is required"); - } - if (!apiKeyId || !apiKey) { - throw new Error("API credentials are required"); - } - - return new DataCollector( - logger, - componentName, - datasetId, - apiKeyId, - apiKey, - ); - } -} -``` - -{{% /tab %}} -{{% tab name="Flutter" %}} - -```dart -import 'dart:typed_data'; -import 'dart:io'; -import 'package:viam_sdk/viam_sdk.dart'; -import 'package:viam_sdk/src/app/app_client.dart'; -import 'package:viam_sdk/src/app/data_client.dart'; -import 'package:image/image.dart' as img; - -const String machinePartId = 'your-machine-part-id-here'; - -class DataCollector { - final Logging logger; - final String componentName; - final String datasetId; - final String apiKeyId; - final String apiKey; - - DataCollector({ - required this.logger, - required this.componentName, - required this.datasetId, - required this.apiKeyId, - required this.apiKey, - }); - - Future captureAndStoreImage( - Image processedImage, - String classification, - ) async { - if (machinePartId.isEmpty) { - throw Exception('Machine part ID not configured'); - } - - DataClient? dataClient; - try { - dataClient = await _createDataClient(); - - final imageData = await _encodeImageToPng(processedImage); - final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; - final filename = '$classification-sample-$timestamp.png'; - - const componentType = 'rdk:component:camera'; - const fileExtension = 'png'; - - final uploadOptions = FileUploadOptions( - componentType: componentType, - componentName: componentName, - fileName: filename, - fileExtension: fileExtension, - ); - - final fileId = await dataClient.fileUploadFromBytes( - machinePartId, - imageData, - uploadOptions, - ); - - await dataClient.addBinaryDataToDatasetByIds( - [fileId], - datasetId, - ); - - logger.info( - 'Successfully added $classification image to dataset $datasetId ' - '(file ID: $fileId, machine: $machinePartId)', - ); - } catch (error) { - logger.error('File upload failed for $classification: $error'); - rethrow; - } finally { - await dataClient?.close(); - } - } - - Future _createDataClient() async { - if (apiKeyId.isEmpty || apiKey.isEmpty) { - throw Exception('API credentials not configured'); - } - - final appClient = await AppClient.withApiKey( - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - - return appClient.dataClient; - } - - Future _encodeImageToPng(Image image) async { - try { - final pngBytes = img.encodePng(image); - return Uint8List.fromList(pngBytes); - } catch (error) { - logger.error('Image encoding failed: $error'); - throw Exception('Failed to encode image: $error'); - } - } - - static DataCollector create({ - required Logging logger, - required String componentName, - required String datasetId, - required String apiKeyId, - required String apiKey, - }) { - if (componentName.isEmpty) { - throw ArgumentError('Component name is required'); - } - if (datasetId.isEmpty) { - throw ArgumentError('Dataset ID is required'); - } - if (apiKeyId.isEmpty || apiKey.isEmpty) { - throw ArgumentError('API credentials are required'); - } - - return DataCollector( - logger: logger, - componentName: componentName, - datasetId: datasetId, - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - } -} -``` - -{{% /tab %}} -{{< /tabs >}} +Once you've captured [enough images for training](/data-ai/train/train-tflite/), you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 91de84e60b..9a4e5e5484 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -139,10 +139,10 @@ final dataClient = viamClient.dataClient; print("Creating dataset..."); try { -final datasetId = await dataClient.createDataset( - name: "", - organizationId: "", -); + final datasetId = await dataClient.createDataset( + name: "", + organizationId: "", + ); print("Created dataset: $datasetId"); } catch (e) { print("Error creating dataset. It may already exist."); diff --git a/docs/data-ai/train/train-tflite.md b/docs/data-ai/train/train-tflite.md index 63cc63708e..d1e38a1a45 100644 --- a/docs/data-ai/train/train-tflite.md +++ b/docs/data-ai/train/train-tflite.md @@ -152,7 +152,7 @@ Using this approach, each subsequent model version becomes more accurate than th To capture images of edge cases and re-train your model using those images, complete the following steps: -1. Add edge case images to your training dataset. You can find edge cases in your existing data on the [**DATA** page](https://app.viam.com/data/) or [capture new images and add them to your training dataset](/data-ai/train/create-dataset/#capture-images). +1. Add edge case images to your training dataset. You can find edge cases in your existing data on the [**DATA** page](https://app.viam.com/data/) or [capture new images and add them to your training dataset](/data-ai/train/capture-images/). 1. Visit the **DATASET** tab of the **DATA** page and annotate the image. diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index a28f2a3919..944efec90e 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -57,343 +57,113 @@ Use the Viam CLI to filter images by label and add the filtered images to a data {{% /tab %}} {{% tab name="Python" %}} -To add an image to a dataset, use [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): ```python -import asyncio -from PIL import Image -from viam.app.app_client import AppClient -from viam.app.data_client import DataClient - -DATASET_ID = "your-dataset-id-here" -API_KEY_ID = "your-api-key-id" -API_KEY = "your-api-key" - - -async def add_image_to_dataset(): - """Add image to Viam dataset using Python SDK""" - - # Initialize app client - app_client = AppClient.create_from_api_key( - api_key_id=API_KEY_ID, - api_key=API_KEY - ) - - try: - # Get data client - data_client = app_client.data_client - - # Load image file - image_path = "sample_image.jpg" - with open(image_path, "rb") as image_file: - image_data = image_file.read() - - # Upload image to Viam cloud - file_upload_metadata = await data_client.file_upload_from_bytes( - part_id=MACHINE_PART_ID, - data=image_data, - component_type="camera", - component_name="camera-1", - file_name="training_sample.jpg", - file_extension="jpg" - ) - - file_id = file_upload_metadata.file_id - print(f"Image uploaded successfully. File ID: {file_id}") - - # Add uploaded file to dataset - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[file_id], - dataset_id=DATASET_ID - ) - - print(f"Image added to dataset {DATASET_ID} successfully") - - except Exception as error: - print(f"Operation failed: {error}") - raise - - finally: - app_client.close() - - -async def main() -> int: - """Main execution function.""" - viam_client = await connect() - data_client = viam_client.data_client - - matching_data = await fetch_binary_data_ids(data_client, PART_ID) - - print("Creating dataset...") - - try: - dataset_id = await data_client.create_dataset( - name=DATASET_NAME, - organization_id=ORG_ID, - ) - print(f"Created dataset: {dataset_id}") - except Exception as e: - print("Error creating dataset. It may already exist.") - print("See: https://app.viam.com/data/datasets") - print(f"Exception: {e}") - return 1 - - print("Adding data to dataset...") - - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[ - obj.metadata.binary_data_id for obj in matching_data - ], - dataset_id=dataset_id - ) +# Connect to Viam client +dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, +) - print("Added files to dataset.") - print( - f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}" - ) +viam_client = await ViamClient.create_from_dial_options(dial_options) +data_client = viam_client.data_client - viam_client.close() - return 0 +# Add image to dataset +await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[EXISTING_IMAGE_ID], + dataset_id=EXISTING_DATASET_ID +) -if __name__ == "__main__": - asyncio.run(main()) +print("Image added to dataset successfully") +viam_client.close() ``` {{% /tab %}} {{% tab name="Go" %}} -To add an image to a dataset, use [`dataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`DataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): ```go -package main - -import ( - "context" - "fmt" - "os" - "time" +ctx := context.Background() +viamClient, err := client.New(ctx, "", logger) +if err != nil { + log.Fatal(err) +} +defer viamClient.Close(ctx) - "go.viam.com/rdk/app/appclient" - "go.viam.com/rdk/services/datamanager/app" -) +dataClient := viamClient.DataClient() -const ( - MachinePartID = "your-machine-part-id-here" - DatasetID = "your-dataset-id-here" - APIKeyID = "your-api-key-id" - APIKey = "your-api-key" +// Add image to dataset +err = dataClient.AddBinaryDataToDatasetByIDs( + context.Background(), + []string{ExistingImageID}, + ExistingDatasetID, ) - -func addImageToDataset() error { - ctx := context.Background() - - // Create app client - client, err := appclient.New(ctx, appclient.Config{ - KeyID: APIKeyID, - Key: APIKey, - }) - if err != nil { - return fmt.Errorf("failed to create app client: %w", err) - } - defer client.Close() - - // Get data client - dataClient := client.DataClient - - // Read image file - imagePath := "sample_image.jpg" - imageData, err := os.ReadFile(imagePath) - if err != nil { - return fmt.Errorf("failed to read image file: %w", err) - } - - // Prepare upload options - componentType := "camera" - componentName := "camera-1" - fileName := fmt.Sprintf("training_sample_%d.jpg", time.Now().Unix()) - fileExtension := "jpg" - - uploadOptions := app.FileUploadOptions{ - ComponentType: &componentType, - ComponentName: &componentName, - FileName: &fileName, - FileExtension: &fileExtension, - } - - // Upload image - fileID, err := dataClient.FileUploadFromBytes( - ctx, - MachinePartID, - imageData, - &uploadOptions, - ) - if err != nil { - return fmt.Errorf("failed to upload image: %w", err) - } - - fmt.Printf("Image uploaded successfully. File ID: %s\n", fileID) - - // Add file to dataset - err = dataClient.AddBinaryDataToDatasetByIDs( - ctx, - []string{fileID}, - DatasetID, - ) - if err != nil { - return fmt.Errorf("failed to add image to dataset: %w", err) - } - - fmt.Printf("Image added to dataset %s successfully\n", DatasetID) - return nil -} - -func main() { - if err := addImageToDataset(); err != nil { - fmt.Printf("Operation failed: %v\n", err) - os.Exit(1) - } +if err != nil { + return fmt.Errorf("failed to add image to dataset: %w", err) } +fmt.Println("Image added to dataset successfully") ``` {{% /tab %}} {{% tab name="TypeScript" %}} -To add an image to a dataset, use [`dataClient.addBinaryDataToDatasetByIds`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): ```typescript -import { createAppClient } from "@viamrobotics/app-client"; -import { promises as fs } from "fs"; - -const MACHINE_PART_ID = "your-machine-part-id-here"; -const DATASET_ID = "your-dataset-id-here"; -const API_KEY_ID = "your-api-key-id"; -const API_KEY = "your-api-key"; - -async function addImageToDataset(): Promise { - // Create app client - const appClient = createAppClient({ - apiKeyId: API_KEY_ID, - apiKey: API_KEY, - }); - - try { - // Get data client - const dataClient = appClient.dataClient(); - - // Read image file - const imagePath = "sample_image.jpg"; - const imageBuffer = await fs.readFile(imagePath); - const imageData = new Uint8Array(imageBuffer); - - // Upload image - const uploadResponse = await dataClient.fileUploadFromBytes({ - partId: MACHINE_PART_ID, - data: imageData, - componentType: "camera", - componentName: "camera-1", - fileName: `training_sample_${Date.now()}.jpg`, - fileExtension: "jpg", - }); - - const fileId = uploadResponse.fileId; - console.log(`Image uploaded successfully. File ID: ${fileId}`); +const client: ViamClient = await createViamClient({ + credential: { + type: "api-key", + payload: API_KEY, + }, + authEntity: API_KEY_ID, +}); - // Add file to dataset - await dataClient.addBinaryDataToDatasetByIds({ - binaryIds: [fileId], - datasetId: DATASET_ID, - }); +const dataClient = client.dataClient; - console.log(`Image added to dataset ${DATASET_ID} successfully`); - } catch (error) { - console.error(`Operation failed: ${error}`); - throw error; - } finally { - await appClient.close(); - } -} - -// Execute function -addImageToDataset().catch((error) => { - console.error("Failed to add image to dataset:", error); - process.exit(1); +// Add image to dataset +await dataClient.addBinaryDataToDatasetByIds({ + binaryIds: [EXISTING_IMAGE_ID], + datasetId: EXISTING_DATASET_ID, }); + +console.log("Image added to dataset successfully"); +client.disconnect(); ``` {{% /tab %}} {{% tab name="Flutter" %}} -To add an image to a dataset, use [`dataClient.addBinaryDataToDatasetByIds`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - -```dart -import 'dart:io'; -import 'dart:typed_data'; -import 'package:viam_sdk/viam_sdk.dart'; - -const String machinePartId = 'your-machine-part-id-here'; -const String datasetId = 'your-dataset-id-here'; -const String apiKeyId = 'your-api-key-id'; -const String apiKey = 'your-api-key'; - -Future addImageToDataset() async { - AppClient? appClient; - - try { - // Create app client - appClient = await AppClient.withApiKey( - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - - // Get data client - final dataClient = appClient.dataClient; - // Read image file - const imagePath = 'sample_image.jpg'; - final imageFile = File(imagePath); - final imageBytes = await imageFile.readAsBytes(); - - // Upload image - final uploadOptions = FileUploadOptions( - componentType: 'camera', - componentName: 'camera-1', - fileName: 'training_sample_${DateTime.now().millisecondsSinceEpoch}.jpg', - fileExtension: 'jpg', - ); +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - final fileId = await dataClient.fileUploadFromBytes( - machinePartId, - imageBytes, - uploadOptions, - ); +```dart +final client = await ViamClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, +); - print('Image uploaded successfully. File ID: $fileId'); +final dataClient = client.dataClient; - // Add file to dataset +try { + // Add image to dataset await dataClient.addBinaryDataToDatasetByIds( - [fileId], - datasetId, + binaryIds: [existingImageId], + datasetId: existingDatasetId, ); - print('Image added to dataset $datasetId successfully'); - - } catch (error) { - print('Operation failed: $error'); - rethrow; - } finally { - await appClient?.close(); - } -} - -void main() async { - try { - await addImageToDataset(); - } catch (error) { - print('Failed to add image to dataset: $error'); - exit(1); - } + print('Image added to dataset successfully'); +} finally { + await client.close(); } ``` @@ -418,13 +188,14 @@ from viam.utils import create_filter # Configuration constants – replace with your actual values DATASET_NAME = "" # a unique, new name for the dataset you want to create ORG_ID = "" # your organization ID, find in your organization settings -PART_ID = "" # ID of machine that captured target images, find in machine config +PART_ID = "" # ID of machine that captured images, find in machine config API_KEY = "" # API key, find or create in your organization settings API_KEY_ID = "" # API key ID, find or create in your organization settings # Adjust the maximum number of images to add to the dataset MAX_MATCHES = 500 + async def connect() -> ViamClient: """Establish a connection to the Viam client using API credentials.""" dial_options = DialOptions( @@ -537,7 +308,10 @@ func connect(ctx context.Context) (*app.ViamClient, error) { return client, nil } -func fetchBinaryDataIDs(ctx context.Context, dataClient data.DataServiceClient, partID string) ([]*data.BinaryData, error) { +func fetchBinaryDataIDs( + ctx context.Context, + dataClient data.DataServiceClient, + partID string) ([]*data.BinaryData, error) { filter := &data.Filter{ PartId: partID, } @@ -840,3 +614,545 @@ Future main() async { {{% /tab %}} {{< /tabs >}} + +## Capture, annotate, and add images to a dataset + +The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. +You can use this logic to expand and improve your datasets continuously over time as your application runs. +Re-train your ML model on the improved dataset to improve the ML model. + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +import asyncio +import os +import time +from typing import Optional +from io import BytesIO + +from PIL import Image +from viam.app.app_client import AppClient +from viam.logging import getLogger + +# Global machine configuration +MACHINE_PART_ID = "your-machine-part-id-here" + + +class DataCollector: + def __init__(self, + component_name: str, dataset_id: str, + api_key_id: str, api_key: str): + + self.logger = getLogger(__name__) + self.component_name = component_name + self.dataset_id = dataset_id + self.api_key_id = api_key_id + self.api_key = api_key + + async def capture_and_store_image(self, + processed_image: Image.Image, + classification: str) -> None: + + if not MACHINE_PART_ID: + raise ValueError("machine part ID not configured") + + # Create fresh data client connection + async with await self._create_data_client() as data_client: + image_data = self._encode_image_to_png(processed_image) + + # Generate unique filename with timestamp + timestamp = int(time.time()) + filename = f"{classification}-sample-{timestamp}.png" + + component_type = "rdk:component:camera" + + upload_metadata = { + "component_type": component_type, + "component_name": self.component_name, + "file_name": filename, + "file_extension": "png" + } + + try: + file_id = await data_client.file_upload_from_bytes( + part_id=MACHINE_PART_ID, + data=image_data, + **upload_metadata + ) + + # Associate file with dataset immediately + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[file_id], + dataset_id=self.dataset_id + ) + + self.logger.info( + f"successfully added {classification} image to dataset " + f"{self.dataset_id} (file ID: {file_id}, " + f"machine: {MACHINE_PART_ID})" + ) + + except Exception as e: + self.logger.error(f"failed to upload and associate image: {e}") + raise + + async def _create_data_client(self) -> AppClient: + if not self.api_key_id or not self.api_key: + raise ValueError("API credentials not configured") + + try: + client = AppClient.create_from_dial_options( + dial_options={ + "auth_entity": self.api_key_id, + "credentials": { + "type": "api-key", + "payload": self.api_key + } + } + ) + return client + + except Exception as e: + raise ValueError(f"failed to create app client: {e}") + + def _encode_image_to_png(self, img: Image.Image) -> bytes: + buffer = BytesIO() + img.save(buffer, format='PNG', optimize=True) + return buffer.getvalue() + + +def create_data_collector( + component_name: str, + dataset_id: str, + api_key_id: str, + api_key: str + ) -> DataCollector: + if not component_name: + raise ValueError("component name is required") + if not dataset_id: + raise ValueError("dataset ID is required") + if not api_key_id or not api_key: + raise ValueError("API credentials are required") + + return DataCollector( + component_name=component_name, + dataset_id=dataset_id, + api_key_id=api_key_id, + api_key=api_key + ) + + +# Example usage +async def main(): + collector = create_data_collector( + component_name="main_camera", + dataset_id="your-dataset-id", + api_key_id="your-api-key-id", + api_key="your-api-key" + ) + + # Example with PIL Image + sample_image = Image.new('RGB', (640, 480), color='red') + await collector.capture_and_store_image(sample_image, "positive_sample") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package datasetcreation + +import ( + "context" + "fmt" + "image" + "time" + + "go.viam.com/rdk/logging" + "go.viam.com/rdk/services/datamanager/app" + "go.viam.com/rdk/app/appclient" +) + +var MachinePartID string = "your-machine-part-id-here" + +type DataCollector struct { + logger logging.Logger + componentName string + datasetID string + apiKeyID string + apiKey string +} + +func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { + if MachinePartID == "" { + return fmt.Errorf("machine part ID not configured") + } + + // Create fresh data client connection + dataClient, err := dc.createDataClient(ctx) + if err != nil { + dc.logger.Errorf("failed to create data client: %v", err) + return fmt.Errorf("data client initialization failed: %w", err) + } + defer dataClient.Close() + + imageData, err := dc.encodeImageToPNG(processedImage) + if err != nil { + dc.logger.Errorf("image encoding failed: %v", err) + return fmt.Errorf("failed to encode image: %w", err) + } + + // Generate unique filename with timestamp + timestamp := time.Now().Unix() + filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) + + componentType := "rdk:component:camera" + fileExtension := "png" + + uploadOptions := app.FileUploadOptions{ + ComponentType: &componentType, + ComponentName: &dc.componentName, + FileName: &filename, + FileExtension: &fileExtension, + } + + fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) + if err != nil { + dc.logger.Errorf("file upload failed for %s: %v", filename, err) + return fmt.Errorf("failed to upload image: %w", err) + } + + // Associate file with dataset immediately + err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) + if err != nil { + dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) + return fmt.Errorf("failed to add image to dataset: %w", err) + } + + dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", + classification, dc.datasetID, fileID, MachinePartID) + + return nil +} + +func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { + if dc.apiKeyID == "" || dc.apiKey == "" { + return nil, fmt.Errorf("API credentials not configured") + } + + client, err := appclient.New(ctx, appclient.Config{ + KeyID: dc.apiKeyID, + Key: dc.apiKey, + }) + if err != nil { + return nil, fmt.Errorf("failed to create app client: %w", err) + } + + return client.DataClient, nil +} + +func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { + // PNG encoding implementation + return nil, nil // Placeholder +} + +func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { + if logger == nil { + return nil, fmt.Errorf("logger is required") + } + if componentName == "" { + return nil, fmt.Errorf("component name is required") + } + if datasetID == "" { + return nil, fmt.Errorf("dataset ID is required") + } + if apiKeyID == "" || apiKey == "" { + return nil, fmt.Errorf("API credentials are required") + } + + return &DataCollector{ + logger: logger, + componentName: componentName, + datasetID: datasetID, + apiKeyID: apiKeyID, + apiKey: apiKey, + }, nil +} +``` + +{{% /tab %}} +{{% tab name="Typescript" %}} + +```typescript +import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; +import { AppClient, DataClient } from "@viamrobotics/app-client"; +import { Logger } from "@viamrobotics/utils"; + +const MACHINE_PART_ID: string = "your-machine-part-id-here"; + +interface FileUploadOptions { + componentType?: string; + componentName?: string; + fileName?: string; + fileExtension?: string; +} + +export class DataCollector { + private logger: Logger; + private componentName: string; + private datasetId: string; + private apiKeyId: string; + private apiKey: string; + + constructor( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string, + ) { + this.logger = logger; + this.componentName = componentName; + this.datasetId = datasetId; + this.apiKeyId = apiKeyId; + this.apiKey = apiKey; + } + + async captureAndStoreImage( + processedImage: ArrayBuffer, + classification: string, + ): Promise { + if (!MACHINE_PART_ID) { + throw new Error("Machine part ID not configured"); + } + + let dataClient: DataClient | null = null; + + try { + dataClient = await this.createDataClient(); + + const imageData = await this.encodeImageToPng(processedImage); + const timestamp = Math.floor(Date.now() / 1000); + const filename = `${classification}-sample-${timestamp}.png`; + + const componentType = "rdk:component:camera"; + const fileExtension = "png"; + + const uploadOptions: FileUploadOptions = { + componentType, + componentName: this.componentName, + fileName: filename, + fileExtension, + }; + + const fileId = await dataClient.fileUploadFromBytes( + MACHINE_PART_ID, + new Uint8Array(imageData), + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); + + this.logger.info( + `Successfully added ${classification} image to dataset ${this.datasetId} ` + + `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, + ); + } catch (error) { + this.logger.error(`File upload failed for ${classification}: ${error}`); + throw new Error(`Failed to upload image: ${error}`); + } finally { + if (dataClient) { + await dataClient.close(); + } + } + } + + private async createDataClient(): Promise { + if (!this.apiKeyId || !this.apiKey) { + throw new Error("API credentials not configured"); + } + + const appClient = new AppClient({ + apiKeyId: this.apiKeyId, + apiKey: this.apiKey, + }); + + return appClient.dataClient(); + } + + private async encodeImageToPng(image: ArrayBuffer): Promise { + try { + // PNG encoding implementation would depend on your image processing library + // This is a placeholder - you would use a library like 'pngjs' or 'canvas' + return image; // Assuming image is already PNG encoded + } catch (error) { + this.logger.error(`Image encoding failed: ${error}`); + throw new Error(`Failed to encode image: ${error}`); + } + } + + static create( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string, + ): DataCollector { + if (!logger) { + throw new Error("Logger is required"); + } + if (!componentName) { + throw new Error("Component name is required"); + } + if (!datasetId) { + throw new Error("Dataset ID is required"); + } + if (!apiKeyId || !apiKey) { + throw new Error("API credentials are required"); + } + + return new DataCollector( + logger, + componentName, + datasetId, + apiKeyId, + apiKey, + ); + } +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'dart:typed_data'; +import 'dart:io'; +import 'package:viam_sdk/viam_sdk.dart'; +import 'package:viam_sdk/src/app/app_client.dart'; +import 'package:viam_sdk/src/app/data_client.dart'; +import 'package:image/image.dart' as img; + +const String machinePartId = 'your-machine-part-id-here'; + +class DataCollector { + final Logging logger; + final String componentName; + final String datasetId; + final String apiKeyId; + final String apiKey; + + DataCollector({ + required this.logger, + required this.componentName, + required this.datasetId, + required this.apiKeyId, + required this.apiKey, + }); + + Future captureAndStoreImage( + Image processedImage, + String classification, + ) async { + if (machinePartId.isEmpty) { + throw Exception('Machine part ID not configured'); + } + + DataClient? dataClient; + try { + dataClient = await _createDataClient(); + + final imageData = await _encodeImageToPng(processedImage); + final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; + final filename = '$classification-sample-$timestamp.png'; + + const componentType = 'rdk:component:camera'; + const fileExtension = 'png'; + + final uploadOptions = FileUploadOptions( + componentType: componentType, + componentName: componentName, + fileName: filename, + fileExtension: fileExtension, + ); + + final fileId = await dataClient.fileUploadFromBytes( + machinePartId, + imageData, + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds( + [fileId], + datasetId, + ); + + logger.info( + 'Successfully added $classification image to dataset $datasetId ' + '(file ID: $fileId, machine: $machinePartId)', + ); + } catch (error) { + logger.error('File upload failed for $classification: $error'); + rethrow; + } finally { + await dataClient?.close(); + } + } + + Future _createDataClient() async { + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw Exception('API credentials not configured'); + } + + final appClient = await AppClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + + return appClient.dataClient; + } + + Future _encodeImageToPng(Image image) async { + try { + final pngBytes = img.encodePng(image); + return Uint8List.fromList(pngBytes); + } catch (error) { + logger.error('Image encoding failed: $error'); + throw Exception('Failed to encode image: $error'); + } + } + + static DataCollector create({ + required Logging logger, + required String componentName, + required String datasetId, + required String apiKeyId, + required String apiKey, + }) { + if (componentName.isEmpty) { + throw ArgumentError('Component name is required'); + } + if (datasetId.isEmpty) { + throw ArgumentError('Dataset ID is required'); + } + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw ArgumentError('API credentials are required'); + } + + return DataCollector( + logger: logger, + componentName: componentName, + datasetId: datasetId, + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + } +} +``` + +{{% /tab %}} +{{< /tabs >}} From 332949cca70803063cac55d959d06ad917ccc309 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 12:27:10 -0400 Subject: [PATCH 11/29] remove duplicate blank lines --- docs/data-ai/train/update-dataset.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 944efec90e..24c910487e 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -142,7 +142,6 @@ client.disconnect(); {{% /tab %}} {{% tab name="Flutter" %}} - To add an image to a dataset, find the binary data ID for the image and the dataset ID. Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): From bebdd3de0cc9a8a8814b3487831c59df8d4ecb71 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 12:28:50 -0400 Subject: [PATCH 12/29] Fix prettier issues --- docs/data-ai/train/capture-images.md | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index 5db329febe..febde12751 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -165,19 +165,19 @@ To capture an image and add it to your **DATA** page, fetch an image from your c Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): ```typescript -const CAMERA_NAME = ''; -const MACHINE_ADDRESS = ''; -const API_KEY = ''; -const API_KEY_ID = ''; -const PART_ID = ''; +const CAMERA_NAME = ""; +const MACHINE_ADDRESS = ""; +const API_KEY = ""; +const API_KEY_ID = ""; +const PART_ID = ""; const machine = await Viam.createRobotClient({ - host: MACHINE_ADDRESS, - credential: { - type: 'api-key', - payload: API_KEY, - }, - authEntity: API_KEY_ID, + host: MACHINE_ADDRESS, + credential: { + type: "api-key", + payload: API_KEY, + }, + authEntity: API_KEY_ID, }); const client: ViamClient = await createViamClient({ @@ -198,13 +198,13 @@ const imageFrame = await camera.getImage(); // Upload binary data const now = new Date(); const fileId = await dataClient.binaryDataCaptureUpload({ - partId: PART_ID, - componentType: 'camera', - componentName: CAMERA_NAME, - methodName: 'GetImages', - dataRequestTimes: [now, now], - fileExtension: '.jpg', - binaryData: imageFrame, + partId: PART_ID, + componentType: "camera", + componentName: CAMERA_NAME, + methodName: "GetImages", + dataRequestTimes: [now, now], + fileExtension: ".jpg", + binaryData: imageFrame, }); // Cleanup From 377d432d9bb89aa56c4d295ab7cba02b0db44d9d Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 13:04:41 -0400 Subject: [PATCH 13/29] fix link --- docs/data-ai/train/capture-images.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index febde12751..d3d3f90a53 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -272,7 +272,7 @@ Once you've captured [enough images for training](/data-ai/train/train-tflite/), To capture a large number of images for training an ML model, use the data management service to [capture and sync image data](/data-ai/capture-data/capture-sync/) from your camera. When you sync with data management, Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. -To use your captured images for training, [annotate the images](data-ai/train/annotate-images/), then [add the images to a dataset](/data-ai/train/update-dataset/). +To use your captured images for training, [annotate the images](/data-ai/train/annotate-images/), then [add the images to a dataset](/data-ai/train/update-dataset/). {{< alert title="Tip" color="tip" >}} From c7276df37049353879a65c714a77fd0c5b4f297c Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 14:56:41 -0400 Subject: [PATCH 14/29] Make it easier to reach subsequent guide pages from create dataset page --- docs/data-ai/_index.md | 10 +++++++- docs/data-ai/train/create-dataset.md | 34 ++++++++++++---------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/docs/data-ai/_index.md b/docs/data-ai/_index.md index 6ac4dbd307..47c8b7f164 100644 --- a/docs/data-ai/_index.md +++ b/docs/data-ai/_index.md @@ -42,11 +42,19 @@ You can also monitor your machines through teleop, power your application logic, {{< /cards >}} {{< /how-to-expand >}} -{{< how-to-expand "Leverage AI" "8" "INTERMEDIATE" "" "data-platform-ai" >}} +{{< how-to-expand "Train an ML model" "8" "INTERMEDIATE" "" "data-platform-ai" >}} {{< cards >}} {{% card link="/data-ai/train/create-dataset/" noimage="true" %}} +{{% card link="/data-ai/train/capture-images/" noimage="true" %}} +{{% card link="/data-ai/train/update-dataset/" noimage="true" %}} +{{% card link="/data-ai/train/annotate-images/" noimage="true" %}} {{% card link="/data-ai/train/train-tflite/" noimage="true" %}} {{% card link="/data-ai/train/train/" noimage="true" %}} +{{< /cards >}} +{{< /how-to-expand >}} + +{{< how-to-expand "Infer with ML models" "8" "INTERMEDIATE" "" "data-platform-ai" >}} +{{< cards >}} {{% card link="/data-ai/ai/deploy/" noimage="true" %}} {{% card link="/data-ai/ai/run-inference/" noimage="true" %}} {{% card link="/data-ai/ai/alert/" noimage="true" %}} diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 9a4e5e5484..432db00f99 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -17,7 +17,8 @@ To train a machine learning model, you will need a dataset. You can create a dataset using the web UI, the CLI, or one of the SDKs: -## Web UI +{{< tabs >}} +{{% tab name="Web UI" %}} 1. Navigate to the **DATA** page and open the [**DATASETS** tab](https://app.viam.com/data/datasets). @@ -29,27 +30,16 @@ You can create a dataset using the web UI, the CLI, or one of the SDKs: 1. Click **Create dataset** to create the dataset. -Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. - -## CLI - -1. First, install the Viam CLI and authenticate: - - {{< readfile "/static/include/how-to/install-cli.md" >}} - -1. [Log in to the CLI](/dev/tools/cli/#authenticate). - -1. Run the following command to create a dataset, replacing the `` and `` placeholders with your organization ID and a unique name for the dataset: - - ```sh {class="command-line" data-prompt="$"} - viam dataset create --org-id= --name= - ``` +{{% /tab %}} +{{% tab name="CLI" %}} -Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. +Run the following [Viam CLI](/dev/tools/cli/) command to create a dataset, replacing the `` and `` placeholders with your organization ID and a unique name for the dataset: -## SDK +```sh {class="command-line" data-prompt="$"} +viam dataset create --org-id= --name= +``` -{{< tabs >}} +{{% /tab %}} {{% tab name="Python" %}} To create a dataset, pass a unique dataset name and organization ID to [`data_client.create_dataset`](/dev/reference/apis/data-client/#createdataset): @@ -155,3 +145,9 @@ try { {{< /tabs >}} Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. + +{{< cards >}} +{{% card link="/data-ai/train/capture-images/" noimage="true" %}} +{{% card link="/data-ai/train/update-dataset/" noimage="true" %}} +{{% card link="/data-ai/train/annotate-images/" noimage="true" %}} +{{< /cards >}} From f8e3f60dbd7d7cb5e3bc3e22c2706c7faecfe8dc Mon Sep 17 00:00:00 2001 From: nathan contino Date: Thu, 26 Jun 2025 15:24:00 -0400 Subject: [PATCH 15/29] Slight rewords to address remaining feedback on annotation SDK snippets --- docs/data-ai/train/annotate-images.md | 37 ++++++++++++++-------- docs/data-ai/train/train-tflite.md | 11 +++++-- docs/data-ai/train/train.md | 4 +-- docs/data-ai/train/upload-external-data.md | 4 +-- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/annotate-images.md index cce546b6fc..9fe93e1dab 100644 --- a/docs/data-ai/train/annotate-images.md +++ b/docs/data-ai/train/annotate-images.md @@ -7,15 +7,7 @@ type: "docs" description: "Annotate images to train a machine learning model." --- -Use the interface on the [**DATA** page](https://app.viam.com/data/view) to annotate your images. -When you label your dataset, include: - -- images with and _without_ the categories you're looking to identify -- a roughly equal number of images for each category -- images from your production environment, including lighting and camera quality -- examples from every angle and distance that you expect the model to handle - -Viam enables you to annotate images for use with the following machine learning methods. +You can either manually add annotations through the Viam web UI, or add annotations with an existing ML model. ## Prerequisites @@ -34,18 +26,26 @@ Follow the guide to configure a [webcam](/operate/reference/components/camera/we ## Classify images with tags Classification determines a descriptive tag or set of tags for an image. -For example, classification could help you identify: +For example, you could use classification to answer the following questions: -- whether an image of a food display appears `full`, `empty`, or `average` -- the quality of manufacturing output: `good` or `bad` -- what combination of toppings exists on a pizza: `pepperoni`, `sausage` and `pepper`, or `pineapple` and `ham` and `mushroom` +- does an image of a food display appear `full`, `empty`, or `average`? +- the quality of manufacturing output `good` or `bad`? +- what combination of toppings exists on a pizza: `pepperoni`, `sausage`, and `pepper`? or `pineapple`, `ham`, and `mushroom`? Viam supports single and multiple label classification. -To create a training set for classification, annotate tags to describe your images. +To create a training dataset for classification, annotate tags to describe your images. + +{{< alert title="Tip" color="tip" >}} + +Unless you already have an ML model that can generate tags for your dataset, use the Web UI to annotate. + +{{< /alert >}} {{< tabs >}} {{% tab name="Web UI" %}} +The [**DATA** page](https://app.viam.com/data/view) provides an interface for annotating images. + To tag an image: 1. Click on an image, then click the **+** next to the **Tags** option. @@ -257,9 +257,17 @@ For example, object detection could help you identify: To create a training set for object detection, annotate bounding boxes to teach your model to identify objects that you want to detect in future images. +{{< alert title="Tip" color="tip" >}} + +Unless you already have an ML model that can generate tags for your dataset, use the Web UI to annotate. + +{{< /alert >}} + {{< tabs >}} {{% tab name="Web UI" %}} +The [**DATA** page](https://app.viam.com/data/view) provides an interface for annotating images. + To label an object with a bounding box: 1. Click on an image, then click the **Annotate** button in right side menu. @@ -271,6 +279,7 @@ To label an object with a bounding box: {{< alert title="Tip" color="tip" >}} Once created, you can move, resize, or delete the bounding box. + {{< /alert >}} Repeat these steps for all images in the dataset. diff --git a/docs/data-ai/train/train-tflite.md b/docs/data-ai/train/train-tflite.md index d1e38a1a45..84d49f126a 100644 --- a/docs/data-ai/train/train-tflite.md +++ b/docs/data-ai/train/train-tflite.md @@ -32,7 +32,7 @@ Follow this guide to use your image data to train an ML model, so that your mach ## Prerequisites -{{% expand "a machine connected to Viam" %}} +{{% expand "A machine connected to Viam" %}} {{% snippet "setup.md" %}} @@ -46,9 +46,16 @@ To train a model, your dataset must contain the following: - At least 80% of the images have labels - For each training label, at least 10 bounding boxes +When you label your dataset, include: + +- images with and _without_ the categories you're looking to identify +- a roughly equal number of images for each category +- images from your production environment, including lighting and camera quality +- examples from every angle and distance that you expect the model to handle + Follow the guide to [create a dataset](/data-ai/train/create-dataset/). -{{% /expand%}} +{{% /expand %}} ## Train a machine learning model diff --git a/docs/data-ai/train/train.md b/docs/data-ai/train/train.md index 610ae073dd..974447709e 100644 --- a/docs/data-ai/train/train.md +++ b/docs/data-ai/train/train.md @@ -26,7 +26,7 @@ If you wish to do this, skip to [Submit a training job](#submit-a-training-job). ## Prerequisites -{{% expand "A dataset with data you can train an ML model on. Click to see instructions." %}} +{{% expand "A dataset that contains training data" %}} For images, follow the instructions to [Create a dataset](/data-ai/train/create-dataset/) to create a dataset and label data. @@ -34,7 +34,7 @@ For other data, use the [Data Client API](/dev/reference/apis/data-client/) from {{% /expand%}} -{{% expand "The Viam CLI. Click to see instructions." %}} +{{% expand "The Viam CLI" %}} You must have the Viam CLI installed to upload training scripts to the registry. diff --git a/docs/data-ai/train/upload-external-data.md b/docs/data-ai/train/upload-external-data.md index a11721438c..457ef41077 100644 --- a/docs/data-ai/train/upload-external-data.md +++ b/docs/data-ai/train/upload-external-data.md @@ -32,13 +32,13 @@ However, if you already have a cache of data you'd like to use with Viam, you ca ### Prerequisites -{{% expand "A running machine connected to Viam. Click to see instructions." %}} +{{% expand "A running machine connected to Viam" %}} {{% snippet "setup-both.md" %}} {{% /expand%}} -{{< expand "Enable data capture and sync on your machine." >}} +{{< expand "Enable data capture and sync on your machine" >}} Add the [data management service](/data-ai/capture-data/capture-sync/#configure-data-capture-and-sync-for-individual-resources): From f6022f811e74854c16fe9932cfcf55db9a8d9b78 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Mon, 30 Jun 2025 08:52:34 -0400 Subject: [PATCH 16/29] Apply suggestions from code review Co-authored-by: Jessamy Taylor <75634662+JessamyT@users.noreply.github.com> --- docs/data-ai/train/capture-images.md | 12 +++++------- docs/data-ai/train/create-dataset.md | 5 +++-- docs/data-ai/train/train-tflite.md | 2 +- docs/data-ai/train/update-dataset.md | 12 ++++++------ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md index d3d3f90a53..4f01c84761 100644 --- a/docs/data-ai/train/capture-images.md +++ b/docs/data-ai/train/capture-images.md @@ -82,7 +82,7 @@ file_id = await data_client.binary_data_capture_upload( part_id=PART_ID, component_type="camera", component_name=CAMERA_NAME, - method_name="GetImages", + method_name="GetImage", data_request_times=[datetime.utcnow(), datetime.utcnow()], file_extension=".jpg", binary_data=image_frame @@ -151,7 +151,7 @@ fileID, err := dataClient.BinaryDataCaptureUpload(ctx, app.BinaryDataCaptureUplo PartID: PART_ID, ComponentType: "camera", ComponentName: CAMERA_NAME, - MethodName: "GetImages", + MethodName: "GetImage", DataRequestTimes: []time.Time{now, now}, FileExtension: ".jpg", BinaryData: img, @@ -201,7 +201,7 @@ const fileId = await dataClient.binaryDataCaptureUpload({ partId: PART_ID, componentType: "camera", componentName: CAMERA_NAME, - methodName: "GetImages", + methodName: "GetImage", dataRequestTimes: [now, now], fileExtension: ".jpg", binaryData: imageFrame, @@ -251,7 +251,7 @@ final fileId = await dataClient.binaryDataCaptureUpload( partId: PART_ID, componentType: 'camera', componentName: CAMERA_NAME, - methodName: 'GetImages', + methodName: 'GetImage', dataRequestTimes: [now, now], fileExtension: '.jpg', binaryData: imageFrame, @@ -272,12 +272,10 @@ Once you've captured [enough images for training](/data-ai/train/train-tflite/), To capture a large number of images for training an ML model, use the data management service to [capture and sync image data](/data-ai/capture-data/capture-sync/) from your camera. When you sync with data management, Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. -To use your captured images for training, [annotate the images](/data-ai/train/annotate-images/), then [add the images to a dataset](/data-ai/train/update-dataset/). +To use your captured images for training, [add the images to a dataset](/data-ai/train/update-dataset/) and [annotate them](/data-ai/train/annotate-images/), so you can use them to train a model. {{< alert title="Tip" color="tip" >}} Once you have enough images, consider disabling data capture to [avoid incurring fees](https://www.viam.com/product/pricing) for capturing large amounts of training data. {{< /alert >}} - -Once you've captured [enough images for training](/data-ai/train/train-tflite/), you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 432db00f99..bc4e01ab61 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -28,7 +28,7 @@ You can create a dataset using the web UI, the CLI, or one of the SDKs: 1. Enter a unique name for the dataset. -1. Click **Create dataset** to create the dataset. +1. Click **Create dataset**. {{% /tab %}} {{% tab name="CLI" %}} @@ -144,7 +144,8 @@ try { {{% /tab %}} {{< /tabs >}} -Once you create a dataset, [capture](/data-ai/train/capture-images/) images, [add](/data-ai/train/update-dataset/) the images to your dataset, and [annotate](/data-ai/train/annotate-images/) the images with training metadata to train your own ML model. +Finish creating a dataset by adding annotated images to it. +You can capture new images or add existing images: {{< cards >}} {{% card link="/data-ai/train/capture-images/" noimage="true" %}} diff --git a/docs/data-ai/train/train-tflite.md b/docs/data-ai/train/train-tflite.md index 84d49f126a..0b9378b21e 100644 --- a/docs/data-ai/train/train-tflite.md +++ b/docs/data-ai/train/train-tflite.md @@ -1,5 +1,5 @@ --- -linkTitle: "Train TFlite model" +linkTitle: "Train TFLite model" title: "Train a TFlite model" weight: 50 type: "docs" diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md index 24c910487e..1fee33b60a 100644 --- a/docs/data-ai/train/update-dataset.md +++ b/docs/data-ai/train/update-dataset.md @@ -1,10 +1,10 @@ --- linkTitle: "Add to a training dataset" -title: "Add to a training dataset" +title: "Add images to a training dataset" weight: 30 layout: "docs" type: "docs" -description: "Update a dataset used to train a machine learning model." +description: "Update a dataset with more images to train a machine learning model." --- ## Prerequisites @@ -174,7 +174,7 @@ try { The following script adds all images captured from a certain machine to a new dataset: {{< tabs >}} -{{% tab name="Python SDK" %}} +{{% tab name="Python" %}} ```python import asyncio @@ -273,7 +273,7 @@ if __name__ == "__main__": ``` {{% /tab %}} -{{% tab name="Go SDK" %}} +{{% tab name="Go" %}} ```go package main @@ -400,7 +400,7 @@ func main() { ``` {{% /tab %}} -{{% tab name="Typescript" %}} +{{% tab name="TypeScript" %}} ```typescript import { ViamClient, createViamClient } from "@viamrobotics/sdk"; @@ -884,7 +884,7 @@ func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, ``` {{% /tab %}} -{{% tab name="Typescript" %}} +{{% tab name="TypeScript" %}} ```typescript import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; From 4205f6ab1ea67a9fa375718cceb3a3c5accc0606 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Mon, 30 Jun 2025 10:53:31 -0400 Subject: [PATCH 17/29] Update docs/data-ai/_index.md Co-authored-by: Naomi Pentrel <5212232+npentrel@users.noreply.github.com> --- docs/data-ai/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/_index.md b/docs/data-ai/_index.md index 47c8b7f164..94090e0981 100644 --- a/docs/data-ai/_index.md +++ b/docs/data-ai/_index.md @@ -42,7 +42,7 @@ You can also monitor your machines through teleop, power your application logic, {{< /cards >}} {{< /how-to-expand >}} -{{< how-to-expand "Train an ML model" "8" "INTERMEDIATE" "" "data-platform-ai" >}} +{{< how-to-expand "Train an ML model" "6" "INTERMEDIATE" "" "data-platform-ai" >}} {{< cards >}} {{% card link="/data-ai/train/create-dataset/" noimage="true" %}} {{% card link="/data-ai/train/capture-images/" noimage="true" %}} From a9c56a7674c992d1d1c13b5e48902c379b011367 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Mon, 30 Jun 2025 14:19:47 -0400 Subject: [PATCH 18/29] Update docs/data-ai/_index.md Co-authored-by: Naomi Pentrel <5212232+npentrel@users.noreply.github.com> --- docs/data-ai/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/_index.md b/docs/data-ai/_index.md index 94090e0981..3e01970c25 100644 --- a/docs/data-ai/_index.md +++ b/docs/data-ai/_index.md @@ -53,7 +53,7 @@ You can also monitor your machines through teleop, power your application logic, {{< /cards >}} {{< /how-to-expand >}} -{{< how-to-expand "Infer with ML models" "8" "INTERMEDIATE" "" "data-platform-ai" >}} +{{< how-to-expand "Infer with ML models" "4" "INTERMEDIATE" "" "data-platform-ai" >}} {{< cards >}} {{% card link="/data-ai/ai/deploy/" noimage="true" %}} {{% card link="/data-ai/ai/run-inference/" noimage="true" %}} From d26e7c0d20042b83edd161f55f3872f3d94d1dba Mon Sep 17 00:00:00 2001 From: nathan contino Date: Mon, 30 Jun 2025 15:16:32 -0400 Subject: [PATCH 19/29] Implement naomi feedback --- docs/data-ai/_index.md | 10 +- ...e-images.md => capture-annotate-images.md} | 286 +++- docs/data-ai/train/capture-images.md | 281 ---- docs/data-ai/train/create-dataset.md | 1169 ++++++++++++++++- docs/data-ai/train/train-tflite.md | 4 +- docs/data-ai/train/update-dataset.md | 1157 ---------------- netlify.toml | 2 +- 7 files changed, 1448 insertions(+), 1461 deletions(-) rename docs/data-ai/train/{annotate-images.md => capture-annotate-images.md} (61%) delete mode 100644 docs/data-ai/train/capture-images.md delete mode 100644 docs/data-ai/train/update-dataset.md diff --git a/docs/data-ai/_index.md b/docs/data-ai/_index.md index 3e01970c25..c002e81b2e 100644 --- a/docs/data-ai/_index.md +++ b/docs/data-ai/_index.md @@ -25,11 +25,12 @@ You can also monitor your machines through teleop, power your application logic,
-{{< how-to-expand "Capture data" "3" "BEGINNER-FRIENDLY" "" "data-platform-capture" >}} +{{< how-to-expand "Capture data" "4" "BEGINNER-FRIENDLY" "" "data-platform-capture" >}} {{< cards >}} {{% card link="/data-ai/capture-data/capture-sync/" noimage="true" %}} {{% card link="/data-ai/capture-data/filter-before-sync/" noimage="true" %}} {{% card link="/data-ai/capture-data/conditional-sync/" noimage="true" %}} +{{% card link="/data-ai/capture-data/lorawan/" noimage="true" %}} {{< /cards >}} {{< /how-to-expand >}} @@ -42,14 +43,13 @@ You can also monitor your machines through teleop, power your application logic, {{< /cards >}} {{< /how-to-expand >}} -{{< how-to-expand "Train an ML model" "6" "INTERMEDIATE" "" "data-platform-ai" >}} +{{< how-to-expand "Train an ML model" "5" "INTERMEDIATE" "" "data-platform-ai" >}} {{< cards >}} {{% card link="/data-ai/train/create-dataset/" noimage="true" %}} -{{% card link="/data-ai/train/capture-images/" noimage="true" %}} -{{% card link="/data-ai/train/update-dataset/" noimage="true" %}} -{{% card link="/data-ai/train/annotate-images/" noimage="true" %}} +{{% card link="/data-ai/train/capture-annotate-images/" noimage="true" %}} {{% card link="/data-ai/train/train-tflite/" noimage="true" %}} {{% card link="/data-ai/train/train/" noimage="true" %}} +{{% card link="/data-ai/train/upload-external-data/" noimage="true" %}} {{< /cards >}} {{< /how-to-expand >}} diff --git a/docs/data-ai/train/annotate-images.md b/docs/data-ai/train/capture-annotate-images.md similarity index 61% rename from docs/data-ai/train/annotate-images.md rename to docs/data-ai/train/capture-annotate-images.md index 9fe93e1dab..0abf25e561 100644 --- a/docs/data-ai/train/annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -1,14 +1,12 @@ --- -linkTitle: "Annotate images for training" -title: "Annotate images for training" -weight: 40 +linkTitle: "Capture and annotate images" +title: "Capture and annotate images for training" +weight: 20 layout: "docs" type: "docs" -description: "Annotate images to train a machine learning model." +description: "Capture images that you can use to train a machine learning model." --- -You can either manually add annotations through the Viam web UI, or add annotations with an existing ML model. - ## Prerequisites {{% expand "A machine connected to Viam" %}} @@ -21,9 +19,275 @@ You can either manually add annotations through the Viam web UI, or add annotati Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). -{{% /expand %}} +{{% /expand%}} + +{{< alert title="Tip" color="tip" >}} + +For the best results, use the same camera for both training data capture and production deployment. + +{{< /alert >}} + +## Capture images + +### Capture individual images + +{{< tabs >}} +{{% tab name="Web UI" %}} + +You can add images to a dataset directly from a camera or vision component feed in the machine's **CONTROL** or **CONFIGURATION** tabs. + +To add an image directly to a dataset from a visual feed, complete the following steps: + +1. Open the **TEST** panel of any camera or vision service component to view a feed of images from the camera. +1. Click the button marked with the camera icon to save the currently displayed image to a dataset: + {{< imgproc src="/components/camera/add_image_to_dataset_button.png" alt="A button marked with the outline of a camera, emphasized in red" resize="800x" style="width:500px" class="imgzoom" >}} +1. Select an existing dataset. +1. Click **Add** to add the image to the selected dataset. +1. When you see a success notification that reads "Saved image to dataset", you have successfully added the image to the dataset. + +To view images added to your dataset, go to the **DATA** page, open the [**DATASETS** tab](https://app.viam.com/data/datasets), then select your dataset. + +{{% /tab %}} +{{% tab name="Python" %}} + +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`data_client.binary_data_capture_upload`](/dev/reference/apis/data-client/#binarydatacaptureupload): + +```python +CAMERA_NAME = "" +MACHINE_ADDRESS = "" + +dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, +) + +robot_opts = RobotClient.Options.with_api_key( + api_key=API_KEY, + api_key_id=API_KEY_ID +) + +viam_client = await ViamClient.create_from_dial_options(dial_options) +data_client = viam_client.data_client + +robot_client = await RobotClient.at_address(ROBOT_ADDRESS, robot_opts) +camera = Camera.from_robot(robot_client, CAMERA_NAME) + +# Capture image +image_frame = await camera.get_image() + +# Upload data +file_id = await data_client.binary_data_capture_upload( + part_id=PART_ID, + component_type="camera", + component_name=CAMERA_NAME, + method_name="GetImage", + data_request_times=[datetime.utcnow(), datetime.utcnow()], + file_extension=".jpg", + binary_data=image_frame +) + +# Cleanup +await robot_client.close() +viam_client.close() +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`DataClient.BinaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): + +```go +const ( + CAMERA_NAME = "" + MACHINE_ADDRESS = "" + API_KEY = "" + API_KEY_ID = "" + PART_ID = "" +) + +ctx := context.Background() +machine, err := client.New( + ctx, + MACHINE_ADDRESS, + logger, + client.WithDialOptions(rpc.WithEntityCredentials( + API_KEY_ID, + rpc.Credentials{ + Type: rpc.CredentialsTypeAPIKey, + Payload: API_KEY, + }, + )), +) +if err != nil { + return "", err +} +defer machine.Close(ctx) + +viamClient, err := client.New(ctx, MACHINE_ADDRESS, logger) +if err != nil { + log.Fatal(err) +} +defer viamClient.Close(ctx) + +dataClient := viamClient.DataClient() + +camera, err := camera.FromRobot(machine, CAMERA_NAME) +if err != nil { + return "", err +} + +// Capture image +img, _, err := camera.GetImage(ctx) +if err != nil { + return "", err +} -## Classify images with tags +// Upload binary data +now := time.Now().UTC() +fileID, err := dataClient.BinaryDataCaptureUpload(ctx, app.BinaryDataCaptureUploadOptions{ + PartID: PART_ID, + ComponentType: "camera", + ComponentName: CAMERA_NAME, + MethodName: "GetImage", + DataRequestTimes: []time.Time{now, now}, + FileExtension: ".jpg", + BinaryData: img, +}) +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): + +```typescript +const CAMERA_NAME = ""; +const MACHINE_ADDRESS = ""; +const API_KEY = ""; +const API_KEY_ID = ""; +const PART_ID = ""; + +const machine = await Viam.createRobotClient({ + host: MACHINE_ADDRESS, + credential: { + type: "api-key", + payload: API_KEY, + }, + authEntity: API_KEY_ID, +}); + +const client: ViamClient = await createViamClient({ + credential: { + type: "api-key", + payload: API_KEY, + }, + authEntity: API_KEY_ID, +}); + +const dataClient = client.dataClient; + +const camera = new Viam.CameraClient(machine, CAMERA_NAME); + +// Capture image +const imageFrame = await camera.getImage(); + +// Upload binary data +const now = new Date(); +const fileId = await dataClient.binaryDataCaptureUpload({ + partId: PART_ID, + componentType: "camera", + componentName: CAMERA_NAME, + methodName: "GetImage", + dataRequestTimes: [now, now], + fileExtension: ".jpg", + binaryData: imageFrame, +}); + +// Cleanup +await machine.disconnect(); +dataClient.close(); +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. +Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): + +```dart +const String CAMERA_NAME = ''; +const String MACHINE_ADDRESS = ''; +const String API_KEY = ''; +const String API_KEY_ID = ''; +const String PART_ID = ''; + +final machine = await RobotClient.atAddress( + MACHINE_ADDRESS, + RobotClientOptions.withApiKey( + apiKey: API_KEY, + apiKeyId: API_KEY_ID, + ), +); + +final client = await ViamClient.withApiKey( + apiKeyId: API_KEY_ID, + apiKey: API_KEY, +); + +final dataClient = client.dataClient; + +final camera = Camera.fromRobot(machine, CAMERA_NAME); + +// Capture image +final imageFrame = await camera.getImage(); + +// Upload binary data +final now = DateTime.now().toUtc(); +final fileId = await dataClient.binaryDataCaptureUpload( + partId: PART_ID, + componentType: 'camera', + componentName: CAMERA_NAME, + methodName: 'GetImage', + dataRequestTimes: [now, now], + fileExtension: '.jpg', + binaryData: imageFrame, +); + +// Cleanup +await robotClient.close(); +dataClient.close(); +``` + +{{% /tab %}} +{{< /tabs >}} + +Once you've captured [enough images for training](/data-ai/train/train-tflite/), you must [annotate](#annotate-images) the images before you can use them to train a model. + +### Capture images over time + +To capture a large number of images for training an ML model, use the data management service to [capture and sync image data](/data-ai/capture-data/capture-sync/) from your camera. + +When you sync with data management, Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. +To use your captured images for training, [add the images to a dataset](/data-ai/train/create-dataset/#add-to-a-dataset) and [annotate them](#annotate-images), so you can use them to train a model. + +{{< alert title="Tip" color="tip" >}} + +Once you have enough images, consider disabling data capture to [avoid incurring fees](https://www.viam.com/product/pricing) for capturing large amounts of training data. + +{{< /alert >}} + + +You can either manually add annotations through the Viam web UI, or add annotations with an existing ML model. + +## Annotate images + +### Classify images with tags Classification determines a descriptive tag or set of tags for an image. For example, you could use classification to answer the following questions: @@ -246,7 +510,7 @@ await dataClient.addTagsToBinaryDataByIds(tags, myIds); Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. -## Detect objects with bounding boxes +### Detect objects with bounding boxes Object detection identifies and determines the location of certain objects in an image. For example, object detection could help you identify: @@ -259,7 +523,8 @@ To create a training set for object detection, annotate bounding boxes to teach {{< alert title="Tip" color="tip" >}} -Unless you already have an ML model that can generate tags for your dataset, use the Web UI to annotate. +To start a new dataset with no preexisting data or model, use the Web UI to annotate tags. +If you have an ML model that can generate tags for your dataset, consider using the model in an SDK to speed up annotation. {{< /alert >}} @@ -470,3 +735,4 @@ for (final detection in detections) { {{< /tabs >}} Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. + diff --git a/docs/data-ai/train/capture-images.md b/docs/data-ai/train/capture-images.md deleted file mode 100644 index 4f01c84761..0000000000 --- a/docs/data-ai/train/capture-images.md +++ /dev/null @@ -1,281 +0,0 @@ ---- -linkTitle: "Capture images for training" -title: "Capture images for training" -weight: 20 -layout: "docs" -type: "docs" -description: "Capture images that you can use to train a machine learning model." ---- - -## Prerequisites - -{{% expand "A machine connected to Viam" %}} - -{{% snippet "setup.md" %}} - -{{% /expand %}} - -{{% expand "A camera, connected to your machine, to capture images" %}} - -Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). - -{{% /expand%}} - -{{< alert title="Tip" color="tip" >}} - -For the best results, use the same camera for both training data capture and production deployment. - -{{< /alert >}} - -## Capture individual images - -{{< tabs >}} -{{% tab name="Web UI" %}} - -You can add images to a dataset directly from a camera or vision component feed in the machine's **CONTROL** or **CONFIGURATION** tabs. - -To add an image directly to a dataset from a visual feed, complete the following steps: - -1. Open the **TEST** panel of any camera or vision service component to view a feed of images from the camera. -1. Click the button marked with the camera icon to save the currently displayed image to a dataset: - {{< imgproc src="/components/camera/add_image_to_dataset_button.png" alt="A button marked with the outline of a camera, emphasized in red" resize="800x" style="width:500px" class="imgzoom" >}} -1. Select an existing dataset. -1. Click **Add** to add the image to the selected dataset. -1. When you see a success notification that reads "Saved image to dataset", you have successfully added the image to the dataset. - -To view images added to your dataset, go to the **DATA** page, open the [**DATASETS** tab](https://app.viam.com/data/datasets), then select your dataset. - -{{% /tab %}} -{{% tab name="Python" %}} - -To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. -Pass that image and an appropriate set of metadata to [`data_client.binary_data_capture_upload`](/dev/reference/apis/data-client/#binarydatacaptureupload): - -```python -CAMERA_NAME = "" -MACHINE_ADDRESS = "" - -dial_options = DialOptions( - credentials=Credentials( - type="api-key", - payload=API_KEY, - ), - auth_entity=API_KEY_ID, -) - -robot_opts = RobotClient.Options.with_api_key( - api_key=API_KEY, - api_key_id=API_KEY_ID -) - -viam_client = await ViamClient.create_from_dial_options(dial_options) -data_client = viam_client.data_client - -robot_client = await RobotClient.at_address(ROBOT_ADDRESS, robot_opts) -camera = Camera.from_robot(robot_client, CAMERA_NAME) - -# Capture image -image_frame = await camera.get_image() - -# Upload data -file_id = await data_client.binary_data_capture_upload( - part_id=PART_ID, - component_type="camera", - component_name=CAMERA_NAME, - method_name="GetImage", - data_request_times=[datetime.utcnow(), datetime.utcnow()], - file_extension=".jpg", - binary_data=image_frame -) - -# Cleanup -await robot_client.close() -viam_client.close() -``` - -{{% /tab %}} -{{% tab name="Go" %}} - -To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. -Pass that image and an appropriate set of metadata to [`DataClient.BinaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): - -```go -const ( - CAMERA_NAME = "" - MACHINE_ADDRESS = "" - API_KEY = "" - API_KEY_ID = "" - PART_ID = "" -) - -ctx := context.Background() -machine, err := client.New( - ctx, - MACHINE_ADDRESS, - logger, - client.WithDialOptions(rpc.WithEntityCredentials( - API_KEY_ID, - rpc.Credentials{ - Type: rpc.CredentialsTypeAPIKey, - Payload: API_KEY, - }, - )), -) -if err != nil { - return "", err -} -defer machine.Close(ctx) - -viamClient, err := client.New(ctx, MACHINE_ADDRESS, logger) -if err != nil { - log.Fatal(err) -} -defer viamClient.Close(ctx) - -dataClient := viamClient.DataClient() - -camera, err := camera.FromRobot(machine, CAMERA_NAME) -if err != nil { - return "", err -} - -// Capture image -img, _, err := camera.GetImage(ctx) -if err != nil { - return "", err -} - -// Upload binary data -now := time.Now().UTC() -fileID, err := dataClient.BinaryDataCaptureUpload(ctx, app.BinaryDataCaptureUploadOptions{ - PartID: PART_ID, - ComponentType: "camera", - ComponentName: CAMERA_NAME, - MethodName: "GetImage", - DataRequestTimes: []time.Time{now, now}, - FileExtension: ".jpg", - BinaryData: img, -}) -``` - -{{% /tab %}} -{{% tab name="TypeScript" %}} - -To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. -Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): - -```typescript -const CAMERA_NAME = ""; -const MACHINE_ADDRESS = ""; -const API_KEY = ""; -const API_KEY_ID = ""; -const PART_ID = ""; - -const machine = await Viam.createRobotClient({ - host: MACHINE_ADDRESS, - credential: { - type: "api-key", - payload: API_KEY, - }, - authEntity: API_KEY_ID, -}); - -const client: ViamClient = await createViamClient({ - credential: { - type: "api-key", - payload: API_KEY, - }, - authEntity: API_KEY_ID, -}); - -const dataClient = client.dataClient; - -const camera = new Viam.CameraClient(machine, CAMERA_NAME); - -// Capture image -const imageFrame = await camera.getImage(); - -// Upload binary data -const now = new Date(); -const fileId = await dataClient.binaryDataCaptureUpload({ - partId: PART_ID, - componentType: "camera", - componentName: CAMERA_NAME, - methodName: "GetImage", - dataRequestTimes: [now, now], - fileExtension: ".jpg", - binaryData: imageFrame, -}); - -// Cleanup -await machine.disconnect(); -dataClient.close(); -``` - -{{% /tab %}} -{{% tab name="Flutter" %}} - -To capture an image and add it to your **DATA** page, fetch an image from your camera through your machine. -Pass that image and an appropriate set of metadata to [`dataClient.binaryDataCaptureUpload`](/dev/reference/apis/data-client/#binarydatacaptureupload): - -```dart -const String CAMERA_NAME = ''; -const String MACHINE_ADDRESS = ''; -const String API_KEY = ''; -const String API_KEY_ID = ''; -const String PART_ID = ''; - -final machine = await RobotClient.atAddress( - MACHINE_ADDRESS, - RobotClientOptions.withApiKey( - apiKey: API_KEY, - apiKeyId: API_KEY_ID, - ), -); - -final client = await ViamClient.withApiKey( - apiKeyId: API_KEY_ID, - apiKey: API_KEY, -); - -final dataClient = client.dataClient; - -final camera = Camera.fromRobot(machine, CAMERA_NAME); - -// Capture image -final imageFrame = await camera.getImage(); - -// Upload binary data -final now = DateTime.now().toUtc(); -final fileId = await dataClient.binaryDataCaptureUpload( - partId: PART_ID, - componentType: 'camera', - componentName: CAMERA_NAME, - methodName: 'GetImage', - dataRequestTimes: [now, now], - fileExtension: '.jpg', - binaryData: imageFrame, -); - -// Cleanup -await robotClient.close(); -dataClient.close(); -``` - -{{% /tab %}} -{{< /tabs >}} - -Once you've captured [enough images for training](/data-ai/train/train-tflite/), you must [annotate](/data-ai/train/annotate-images/) the images before you can use them to train a model. - -## Capture images over time - -To capture a large number of images for training an ML model, use the data management service to [capture and sync image data](/data-ai/capture-data/capture-sync/) from your camera. - -When you sync with data management, Viam stores the images saved by capture and sync on the [**DATA** page](https://app.viam.com/data/), but does not add the images to a dataset. -To use your captured images for training, [add the images to a dataset](/data-ai/train/update-dataset/) and [annotate them](/data-ai/train/annotate-images/), so you can use them to train a model. - -{{< alert title="Tip" color="tip" >}} - -Once you have enough images, consider disabling data capture to [avoid incurring fees](https://www.viam.com/product/pricing) for capturing large amounts of training data. - -{{< /alert >}} diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index bc4e01ab61..be57fdd02c 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -15,6 +15,8 @@ aliases: To train a machine learning model, you will need a dataset. +## Create a dataset + You can create a dataset using the web UI, the CLI, or one of the SDKs: {{< tabs >}} @@ -147,8 +149,1165 @@ try { Finish creating a dataset by adding annotated images to it. You can capture new images or add existing images: -{{< cards >}} -{{% card link="/data-ai/train/capture-images/" noimage="true" %}} -{{% card link="/data-ai/train/update-dataset/" noimage="true" %}} -{{% card link="/data-ai/train/annotate-images/" noimage="true" %}} -{{< /cards >}} +## Add to a dataset + +{{< tabs >}} +{{% tab name="Web UI" %}} + +You can add images to a dataset from the **Images** tab of the [**DATA** page](https://app.viam.com/data/view): + +1. Click to select the images you would like to add to your dataset. + +1. Click the **Add to dataset** button in the top right. + +1. From the **Dataset** dropdown, select the name of your dataset. + +1. Click **Add \ images** to add the selected images to the dataset. + +{{< alert title="Tip" color="tip" >}} + +To select a range of images, select one image, then hold **Ctrl/Cmd** while clicking another image. +This will select both images as well as the entire range of images between those images. + +{{< /alert >}} + +{{% /tab %}} +{{% tab name="CLI" %}} + +Use the Viam CLI to filter images by label and add the filtered images to a dataset: + +1. First, [create a dataset](/data-ai/train/create-dataset/), if you haven't already. + +1. If you just created a dataset, use the dataset ID output by the creation command. + If your dataset already exists, run the following command to get a list of dataset names and corresponding IDs: + + ```sh {class="command-line" data-prompt="$"} + viam dataset list + ``` + +1. Run the following [command](/dev/tools/cli/#dataset) to add all images labeled with a subset of tags to the dataset, replacing the `` placeholder with the dataset ID output by the command in the previous step: + + ```sh {class="command-line" data-prompt="$"} + viam dataset data add filter --dataset-id= --tags=red_star,blue_square + ``` + +{{% /tab %}} +{{% tab name="Python" %}} + +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + +```python +# Connect to Viam client +dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, +) + +viam_client = await ViamClient.create_from_dial_options(dial_options) +data_client = viam_client.data_client + +# Add image to dataset +await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[EXISTING_IMAGE_ID], + dataset_id=EXISTING_DATASET_ID +) + +print("Image added to dataset successfully") +viam_client.close() +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`DataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + +```go +ctx := context.Background() +viamClient, err := client.New(ctx, "", logger) +if err != nil { + log.Fatal(err) +} +defer viamClient.Close(ctx) + +dataClient := viamClient.DataClient() + +// Add image to dataset +err = dataClient.AddBinaryDataToDatasetByIDs( + context.Background(), + []string{ExistingImageID}, + ExistingDatasetID, +) +if err != nil { + return fmt.Errorf("failed to add image to dataset: %w", err) +} + +fmt.Println("Image added to dataset successfully") +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + +```typescript +const client: ViamClient = await createViamClient({ + credential: { + type: "api-key", + payload: API_KEY, + }, + authEntity: API_KEY_ID, +}); + +const dataClient = client.dataClient; + +// Add image to dataset +await dataClient.addBinaryDataToDatasetByIds({ + binaryIds: [EXISTING_IMAGE_ID], + datasetId: EXISTING_DATASET_ID, +}); + +console.log("Image added to dataset successfully"); +client.disconnect(); +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +To add an image to a dataset, find the binary data ID for the image and the dataset ID. +Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): + +```dart +final client = await ViamClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, +); + +final dataClient = client.dataClient; + +try { + // Add image to dataset + await dataClient.addBinaryDataToDatasetByIds( + binaryIds: [existingImageId], + datasetId: existingDatasetId, + ); + + print('Image added to dataset successfully'); +} finally { + await client.close(); +} +``` + +{{% /tab %}} +{{< /tabs >}} + +## Add all images captured by a specific machine to a dataset + +The following script adds all images captured from a certain machine to a new dataset: + +{{< tabs >}} +{{% tab name="Web UI" %}} + +You can add images to a dataset from the **Images** tab of the [**DATA** page](https://app.viam.com/data/view): + +1. From the **Machine name** dropdown, select the name of a machine. +1. Click the **Apply** button at the bottom of the left sidebar. +1. Click to select the images you would like to add to your dataset. +1. Click the **Add to dataset** button in the top right. +1. From the **Dataset** dropdown, select the name of your dataset. +1. Click **Add \ images** to add the selected images to the dataset. + +{{< alert title="Tip" color="tip" >}} + +To select a range of images, select one image, then hold **Ctrl/Cmd** while clicking another image. +This will select both images as well as the entire range of images between those images. + +{{< /alert >}} + +{{% /tab %}} +{{% tab name="Python" %}} + +```python +import asyncio +from typing import List, Optional + +from viam.rpc.dial import DialOptions, Credentials +from viam.app.viam_client import ViamClient +from viam.utils import create_filter + +# Configuration constants – replace with your actual values +DATASET_NAME = "" # a unique, new name for the dataset you want to create +ORG_ID = "" # your organization ID, find in your organization settings +PART_ID = "" # ID of machine that captured images, find in machine config +API_KEY = "" # API key, find or create in your organization settings +API_KEY_ID = "" # API key ID, find or create in your organization settings + +# Adjust the maximum number of images to add to the dataset +MAX_MATCHES = 500 + + +async def connect() -> ViamClient: + """Establish a connection to the Viam client using API credentials.""" + dial_options = DialOptions( + credentials=Credentials( + type="api-key", + payload=API_KEY, + ), + auth_entity=API_KEY_ID, + ) + return await ViamClient.create_from_dial_options(dial_options) + + +async def fetch_binary_data_ids(data_client, part_id: str) -> List[str]: + """Fetch binary data metadata and return a list of BinaryData objects.""" + data_filter = create_filter(part_id=part_id) + all_matches = [] + last: Optional[str] = None + + print("Getting data for part...") + + while len(all_matches) < MAX_MATCHES: + print("Fetching more data...") + data, _, last = await data_client.binary_data_by_filter( + data_filter, + limit=50, + last=last, + include_binary_data=False, + ) + if not data: + break + all_matches.extend(data) + + return all_matches + + +async def main() -> int: + """Main execution function.""" + viam_client = await connect() + data_client = viam_client.data_client + + matching_data = await fetch_binary_data_ids(data_client, PART_ID) + + print("Creating dataset...") + + try: + dataset_id = await data_client.create_dataset( + name=DATASET_NAME, + organization_id=ORG_ID, + ) + print(f"Created dataset: {dataset_id}") + except Exception as e: + print("Error creating dataset. It may already exist.") + print("See: https://app.viam.com/data/datasets") + print(f"Exception: {e}") + return 1 + + print("Adding data to dataset...") + + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[ + obj.metadata.binary_data_id for obj in matching_data + ], + dataset_id=dataset_id + ) + + print("Added files to dataset.") + print( + f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}" + ) + + viam_client.close() + return 0 + +if __name__ == "__main__": + asyncio.run(main()) +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package main + +import ( + "context" + "fmt" + "log" + + "go.viam.com/rdk/rpc" + "go.viam.com/utils" + "go.viam.com/rdk/app" + "go.viam.com/rdk/app/data" +) + +// Configuration constants - replace with your actual values +const ( + DATASET_NAME = "" // a unique, new name for the dataset you want to create + ORG_ID = "" // your organization ID, find in your organization settings + PART_ID = "" // ID of machine that captured target images, find in machine config + API_KEY = "" // API key, find or create in your organization settings + API_KEY_ID = "" // API key ID, find or create in your organization settings + MAX_MATCHES = 500 +) + +func connect(ctx context.Context) (*app.ViamClient, error) { + client, err := app.NewViamClientWithAPIKey(ctx, API_KEY, API_KEY_ID) + if err != nil { + return nil, fmt.Errorf("failed to create client: %w", err) + } + return client, nil +} + +func fetchBinaryDataIDs( + ctx context.Context, + dataClient data.DataServiceClient, + partID string) ([]*data.BinaryData, error) { + filter := &data.Filter{ + PartId: partID, + } + + var allMatches []*data.BinaryData + var last string + + fmt.Println("Getting data for part...") + + for len(allMatches) < MAX_MATCHES { + fmt.Println("Fetching more data...") + + resp, err := dataClient.BinaryDataByFilter(ctx, &data.BinaryDataByFilterRequest{ + DataRequest: &data.DataRequest{ + Filter: filter, + Limit: 50, + Last: last, + IncludeBinaryData: false, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch binary data: %w", err) + } + + if len(resp.Data) == 0 { + break + } + + allMatches = append(allMatches, resp.Data...) + last = resp.Last + } + + return allMatches, nil +} + +func main() { + ctx := context.Background() + + viamClient, err := connect(ctx) + if err != nil { + log.Fatalf("Connection failed: %v", err) + } + defer viamClient.Close() + + dataClient := viamClient.DataClient + + matchingData, err := fetchBinaryDataIDs(ctx, dataClient, PART_ID) + if err != nil { + log.Fatalf("Failed to fetch binary data: %v", err) + } + + fmt.Println("Creating dataset...") + + datasetResp, err := dataClient.CreateDataset(ctx, &data.CreateDatasetRequest{ + Name: DATASET_NAME, + OrganizationId: ORG_ID, + }) + if err != nil { + fmt.Println("Error creating dataset. It may already exist.") + fmt.Println("See: https://app.viam.com/data/datasets") + fmt.Printf("Exception: %v\n", err) + return + } + + datasetID := datasetResp.Id + fmt.Printf("Created dataset: %s\n", datasetID) + + fmt.Println("Adding data to dataset...") + + var binaryIDs []string + for _, obj := range matchingData { + binaryIDs = append(binaryIDs, obj.Metadata.Id) + } + + _, err = dataClient.AddBinaryDataToDatasetByIds(ctx, &data.AddBinaryDataToDatasetByIdsRequest{ + BinaryIds: binaryIDs, + DatasetId: datasetID, + }) + if err != nil { + log.Fatalf("Failed to add binary data to dataset: %v", err) + } + + fmt.Println("Added files to dataset.") + fmt.Printf("See dataset: https://app.viam.com/data/datasets?id=%s\n", datasetID) +} + +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +import { ViamClient, createViamClient } from "@viamrobotics/sdk"; +import { DataServiceClient } from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb_service"; +import { + BinaryDataByFilterRequest, + CreateDatasetRequest, + AddBinaryDataToDatasetByIdsRequest, + Filter, + DataRequest, +} from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb"; + +// Configuration constants - replace with your actual values +const DATASET_NAME = ""; // a unique, new name for the dataset you want to create +const ORG_ID = ""; // your organization ID, find in your organization settings +const PART_ID = ""; // ID of machine that captured target images, find in machine config +const API_KEY = ""; // API key, find or create in your organization settings +const API_KEY_ID = ""; // API key ID, find or create in your organization settings +const MAX_MATCHES = 500; + +async function connect(): Promise { + return await createViamClient({ + credential: { + type: "api-key", + authEntity: API_KEY_ID, + payload: API_KEY, + }, + }); +} + +async function fetchBinaryDataIds( + dataClient: DataServiceClient, + partId: string, +): Promise { + const filter = new Filter(); + filter.setPartId(partId); + + const allMatches: any[] = []; + let last = ""; + + console.log("Getting data for part..."); + + while (allMatches.length < MAX_MATCHES) { + console.log("Fetching more data..."); + + const dataRequest = new DataRequest(); + dataRequest.setFilter(filter); + dataRequest.setLimit(50); + dataRequest.setLast(last); + dataRequest.setIncludeBinaryData(false); + + const request = new BinaryDataByFilterRequest(); + request.setDataRequest(dataRequest); + + const response = await dataClient.binaryDataByFilter(request); + const data = response.getDataList(); + + if (data.length === 0) { + break; + } + + allMatches.push(...data); + last = response.getLast(); + } + + return allMatches; +} + +async function main(): Promise { + const viamClient = await connect(); + const dataClient = viamClient.dataClient; + + const matchingData = await fetchBinaryDataIds(dataClient, PART_ID); + + console.log("Creating dataset..."); + + try { + const createRequest = new CreateDatasetRequest(); + createRequest.setName(DATASET_NAME); + createRequest.setOrganizationId(ORG_ID); + + const datasetResponse = await dataClient.createDataset(createRequest); + const datasetId = datasetResponse.getId(); + console.log(`Created dataset: ${datasetId}`); + + console.log("Adding data to dataset..."); + + const binaryIds = matchingData.map( + (obj) => obj.getMetadata()?.getId() || "", + ); + + const addRequest = new AddBinaryDataToDatasetByIdsRequest(); + addRequest.setBinaryIdsList(binaryIds); + addRequest.setDatasetId(datasetId); + + await dataClient.addBinaryDataToDatasetByIds(addRequest); + + console.log("Added files to dataset."); + console.log( + `See dataset: https://app.viam.com/data/datasets?id=${datasetId}`, + ); + + viamClient.disconnect(); + return 0; + } catch (error) { + console.log("Error creating dataset. It may already exist."); + console.log("See: https://app.viam.com/data/datasets"); + console.log(`Exception: ${error}`); + viamClient.disconnect(); + return 1; + } +} + +if (require.main === module) { + main().then((code) => process.exit(code)); +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'package:viam_sdk/viam_sdk.dart'; + +// Configuration constants - replace with your actual values +const String datasetName = ""; // a unique, new name for the dataset you want to create +const String orgId = ""; // your organization ID, find in your organization settings +const String partId = ""; // ID of machine that captured target images, find in machine config +const String apiKey = ""; // API key, find or create in your organization settings +const String apiKeyId = ""; // API key ID, find or create in your organization settings +const int maxMatches = 500; + +Future connect() async { + return await ViamClient.withApiKey( + apiKey: apiKey, + apiKeyId: apiKeyId, + ); +} + +Future> fetchBinaryDataIds( + DataClient dataClient, String partId) async { + final filter = Filter(partId: partId); + final List allMatches = []; + String? last; + + print("Getting data for part..."); + + while (allMatches.length < maxMatches) { + print("Fetching more data..."); + + final response = await dataClient.binaryDataByFilter( + filter: filter, + limit: 50, + last: last, + includeBinaryData: false, + ); + + if (response.data.isEmpty) { + break; + } + + allMatches.addAll(response.data); + last = response.last; + } + + return allMatches; +} + +Future main() async { + final viamClient = await connect(); + final dataClient = viamClient.dataClient; + + final matchingData = await fetchBinaryDataIds(dataClient, partId); + + print("Creating dataset..."); + + try { + final datasetId = await dataClient.createDataset( + name: datasetName, + organizationId: orgId, + ); + print("Created dataset: $datasetId"); + + print("Adding data to dataset..."); + + final binaryIds = matchingData + .map((obj) => obj.metadata.binaryDataId) + .toList(); + + await dataClient.addBinaryDataToDatasetByIds( + binaryIds: binaryIds, + datasetId: datasetId, + ); + + print("Added files to dataset."); + print("See dataset: https://app.viam.com/data/datasets?id=$datasetId"); + + viamClient.close(); + return 0; + + } catch (error) { + print("Error creating dataset. It may already exist."); + print("See: https://app.viam.com/data/datasets"); + print("Exception: $error"); + viamClient.close(); + return 1; + } +} +``` + +{{% /tab %}} +{{< /tabs >}} + +## Capture, annotate, and add images to a dataset + +The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. +You can use this logic to expand and improve your datasets continuously over time as your application runs. +Re-train your ML model on the improved dataset to improve the ML model. + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +import asyncio +import os +import time +from typing import Optional +from io import BytesIO + +from PIL import Image +from viam.app.app_client import AppClient +from viam.logging import getLogger + +# Global machine configuration +MACHINE_PART_ID = "your-machine-part-id-here" + + +class DataCollector: + def __init__(self, + component_name: str, dataset_id: str, + api_key_id: str, api_key: str): + + self.logger = getLogger(__name__) + self.component_name = component_name + self.dataset_id = dataset_id + self.api_key_id = api_key_id + self.api_key = api_key + + async def capture_and_store_image(self, + processed_image: Image.Image, + classification: str) -> None: + + if not MACHINE_PART_ID: + raise ValueError("machine part ID not configured") + + # Create fresh data client connection + async with await self._create_data_client() as data_client: + image_data = self._encode_image_to_png(processed_image) + + # Generate unique filename with timestamp + timestamp = int(time.time()) + filename = f"{classification}-sample-{timestamp}.png" + + component_type = "rdk:component:camera" + + upload_metadata = { + "component_type": component_type, + "component_name": self.component_name, + "file_name": filename, + "file_extension": "png" + } + + try: + file_id = await data_client.file_upload_from_bytes( + part_id=MACHINE_PART_ID, + data=image_data, + **upload_metadata + ) + + # Associate file with dataset immediately + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[file_id], + dataset_id=self.dataset_id + ) + + self.logger.info( + f"successfully added {classification} image to dataset " + f"{self.dataset_id} (file ID: {file_id}, " + f"machine: {MACHINE_PART_ID})" + ) + + except Exception as e: + self.logger.error(f"failed to upload and associate image: {e}") + raise + + async def _create_data_client(self) -> AppClient: + if not self.api_key_id or not self.api_key: + raise ValueError("API credentials not configured") + + try: + client = AppClient.create_from_dial_options( + dial_options={ + "auth_entity": self.api_key_id, + "credentials": { + "type": "api-key", + "payload": self.api_key + } + } + ) + return client + + except Exception as e: + raise ValueError(f"failed to create app client: {e}") + + def _encode_image_to_png(self, img: Image.Image) -> bytes: + buffer = BytesIO() + img.save(buffer, format='PNG', optimize=True) + return buffer.getvalue() + + +def create_data_collector( + component_name: str, + dataset_id: str, + api_key_id: str, + api_key: str + ) -> DataCollector: + if not component_name: + raise ValueError("component name is required") + if not dataset_id: + raise ValueError("dataset ID is required") + if not api_key_id or not api_key: + raise ValueError("API credentials are required") + + return DataCollector( + component_name=component_name, + dataset_id=dataset_id, + api_key_id=api_key_id, + api_key=api_key + ) + + +# Example usage +async def main(): + collector = create_data_collector( + component_name="main_camera", + dataset_id="your-dataset-id", + api_key_id="your-api-key-id", + api_key="your-api-key" + ) + + # Example with PIL Image + sample_image = Image.new('RGB', (640, 480), color='red') + await collector.capture_and_store_image(sample_image, "positive_sample") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package datasetcreation + +import ( + "context" + "fmt" + "image" + "time" + + "go.viam.com/rdk/logging" + "go.viam.com/rdk/services/datamanager/app" + "go.viam.com/rdk/app/appclient" +) + +var MachinePartID string = "your-machine-part-id-here" + +type DataCollector struct { + logger logging.Logger + componentName string + datasetID string + apiKeyID string + apiKey string +} + +func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { + if MachinePartID == "" { + return fmt.Errorf("machine part ID not configured") + } + + // Create fresh data client connection + dataClient, err := dc.createDataClient(ctx) + if err != nil { + dc.logger.Errorf("failed to create data client: %v", err) + return fmt.Errorf("data client initialization failed: %w", err) + } + defer dataClient.Close() + + imageData, err := dc.encodeImageToPNG(processedImage) + if err != nil { + dc.logger.Errorf("image encoding failed: %v", err) + return fmt.Errorf("failed to encode image: %w", err) + } + + // Generate unique filename with timestamp + timestamp := time.Now().Unix() + filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) + + componentType := "rdk:component:camera" + fileExtension := "png" + + uploadOptions := app.FileUploadOptions{ + ComponentType: &componentType, + ComponentName: &dc.componentName, + FileName: &filename, + FileExtension: &fileExtension, + } + + fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) + if err != nil { + dc.logger.Errorf("file upload failed for %s: %v", filename, err) + return fmt.Errorf("failed to upload image: %w", err) + } + + // Associate file with dataset immediately + err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) + if err != nil { + dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) + return fmt.Errorf("failed to add image to dataset: %w", err) + } + + dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", + classification, dc.datasetID, fileID, MachinePartID) + + return nil +} + +func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { + if dc.apiKeyID == "" || dc.apiKey == "" { + return nil, fmt.Errorf("API credentials not configured") + } + + client, err := appclient.New(ctx, appclient.Config{ + KeyID: dc.apiKeyID, + Key: dc.apiKey, + }) + if err != nil { + return nil, fmt.Errorf("failed to create app client: %w", err) + } + + return client.DataClient, nil +} + +func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { + // PNG encoding implementation + return nil, nil // Placeholder +} + +func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { + if logger == nil { + return nil, fmt.Errorf("logger is required") + } + if componentName == "" { + return nil, fmt.Errorf("component name is required") + } + if datasetID == "" { + return nil, fmt.Errorf("dataset ID is required") + } + if apiKeyID == "" || apiKey == "" { + return nil, fmt.Errorf("API credentials are required") + } + + return &DataCollector{ + logger: logger, + componentName: componentName, + datasetID: datasetID, + apiKeyID: apiKeyID, + apiKey: apiKey, + }, nil +} +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; +import { AppClient, DataClient } from "@viamrobotics/app-client"; +import { Logger } from "@viamrobotics/utils"; + +const MACHINE_PART_ID: string = "your-machine-part-id-here"; + +interface FileUploadOptions { + componentType?: string; + componentName?: string; + fileName?: string; + fileExtension?: string; +} + +export class DataCollector { + private logger: Logger; + private componentName: string; + private datasetId: string; + private apiKeyId: string; + private apiKey: string; + + constructor( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string, + ) { + this.logger = logger; + this.componentName = componentName; + this.datasetId = datasetId; + this.apiKeyId = apiKeyId; + this.apiKey = apiKey; + } + + async captureAndStoreImage( + processedImage: ArrayBuffer, + classification: string, + ): Promise { + if (!MACHINE_PART_ID) { + throw new Error("Machine part ID not configured"); + } + + let dataClient: DataClient | null = null; + + try { + dataClient = await this.createDataClient(); + + const imageData = await this.encodeImageToPng(processedImage); + const timestamp = Math.floor(Date.now() / 1000); + const filename = `${classification}-sample-${timestamp}.png`; + + const componentType = "rdk:component:camera"; + const fileExtension = "png"; + + const uploadOptions: FileUploadOptions = { + componentType, + componentName: this.componentName, + fileName: filename, + fileExtension, + }; + + const fileId = await dataClient.fileUploadFromBytes( + MACHINE_PART_ID, + new Uint8Array(imageData), + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); + + this.logger.info( + `Successfully added ${classification} image to dataset ${this.datasetId} ` + + `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, + ); + } catch (error) { + this.logger.error(`File upload failed for ${classification}: ${error}`); + throw new Error(`Failed to upload image: ${error}`); + } finally { + if (dataClient) { + await dataClient.close(); + } + } + } + + private async createDataClient(): Promise { + if (!this.apiKeyId || !this.apiKey) { + throw new Error("API credentials not configured"); + } + + const appClient = new AppClient({ + apiKeyId: this.apiKeyId, + apiKey: this.apiKey, + }); + + return appClient.dataClient(); + } + + private async encodeImageToPng(image: ArrayBuffer): Promise { + try { + // PNG encoding implementation would depend on your image processing library + // This is a placeholder - you would use a library like 'pngjs' or 'canvas' + return image; // Assuming image is already PNG encoded + } catch (error) { + this.logger.error(`Image encoding failed: ${error}`); + throw new Error(`Failed to encode image: ${error}`); + } + } + + static create( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string, + ): DataCollector { + if (!logger) { + throw new Error("Logger is required"); + } + if (!componentName) { + throw new Error("Component name is required"); + } + if (!datasetId) { + throw new Error("Dataset ID is required"); + } + if (!apiKeyId || !apiKey) { + throw new Error("API credentials are required"); + } + + return new DataCollector( + logger, + componentName, + datasetId, + apiKeyId, + apiKey, + ); + } +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'dart:typed_data'; +import 'dart:io'; +import 'package:viam_sdk/viam_sdk.dart'; +import 'package:viam_sdk/src/app/app_client.dart'; +import 'package:viam_sdk/src/app/data_client.dart'; +import 'package:image/image.dart' as img; + +const String machinePartId = 'your-machine-part-id-here'; + +class DataCollector { + final Logging logger; + final String componentName; + final String datasetId; + final String apiKeyId; + final String apiKey; + + DataCollector({ + required this.logger, + required this.componentName, + required this.datasetId, + required this.apiKeyId, + required this.apiKey, + }); + + Future captureAndStoreImage( + Image processedImage, + String classification, + ) async { + if (machinePartId.isEmpty) { + throw Exception('Machine part ID not configured'); + } + + DataClient? dataClient; + try { + dataClient = await _createDataClient(); + + final imageData = await _encodeImageToPng(processedImage); + final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; + final filename = '$classification-sample-$timestamp.png'; + + const componentType = 'rdk:component:camera'; + const fileExtension = 'png'; + + final uploadOptions = FileUploadOptions( + componentType: componentType, + componentName: componentName, + fileName: filename, + fileExtension: fileExtension, + ); + + final fileId = await dataClient.fileUploadFromBytes( + machinePartId, + imageData, + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds( + [fileId], + datasetId, + ); + + logger.info( + 'Successfully added $classification image to dataset $datasetId ' + '(file ID: $fileId, machine: $machinePartId)', + ); + } catch (error) { + logger.error('File upload failed for $classification: $error'); + rethrow; + } finally { + await dataClient?.close(); + } + } + + Future _createDataClient() async { + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw Exception('API credentials not configured'); + } + + final appClient = await AppClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + + return appClient.dataClient; + } + + Future _encodeImageToPng(Image image) async { + try { + final pngBytes = img.encodePng(image); + return Uint8List.fromList(pngBytes); + } catch (error) { + logger.error('Image encoding failed: $error'); + throw Exception('Failed to encode image: $error'); + } + } + + static DataCollector create({ + required Logging logger, + required String componentName, + required String datasetId, + required String apiKeyId, + required String apiKey, + }) { + if (componentName.isEmpty) { + throw ArgumentError('Component name is required'); + } + if (datasetId.isEmpty) { + throw ArgumentError('Dataset ID is required'); + } + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw ArgumentError('API credentials are required'); + } + + return DataCollector( + logger: logger, + componentName: componentName, + datasetId: datasetId, + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + } +} +``` + +{{% /tab %}} +{{< /tabs >}} diff --git a/docs/data-ai/train/train-tflite.md b/docs/data-ai/train/train-tflite.md index 0b9378b21e..30045ec4d6 100644 --- a/docs/data-ai/train/train-tflite.md +++ b/docs/data-ai/train/train-tflite.md @@ -44,7 +44,7 @@ To train a model, your dataset must contain the following: - At least 15 images - At least 80% of the images have labels -- For each training label, at least 10 bounding boxes +- For each training label, at least 10 examples When you label your dataset, include: @@ -159,7 +159,7 @@ Using this approach, each subsequent model version becomes more accurate than th To capture images of edge cases and re-train your model using those images, complete the following steps: -1. Add edge case images to your training dataset. You can find edge cases in your existing data on the [**DATA** page](https://app.viam.com/data/) or [capture new images and add them to your training dataset](/data-ai/train/capture-images/). +1. Add edge case images to your training dataset. You can find edge cases in your existing data on the [**DATA** page](https://app.viam.com/data/) or [capture new images and add them to your training dataset](/data-ai/train/capture-annotate-images/). 1. Visit the **DATASET** tab of the **DATA** page and annotate the image. diff --git a/docs/data-ai/train/update-dataset.md b/docs/data-ai/train/update-dataset.md deleted file mode 100644 index 1fee33b60a..0000000000 --- a/docs/data-ai/train/update-dataset.md +++ /dev/null @@ -1,1157 +0,0 @@ ---- -linkTitle: "Add to a training dataset" -title: "Add images to a training dataset" -weight: 30 -layout: "docs" -type: "docs" -description: "Update a dataset with more images to train a machine learning model." ---- - -## Prerequisites - -{{% expand "A machine connected to Viam" %}} - -{{% snippet "setup.md" %}} - -{{% /expand %}} - -{{% expand "A camera, connected to your machine, to capture images" %}} - -Follow the guide to configure a [webcam](/operate/reference/components/camera/webcam/) or similar [camera component](/operate/reference/components/camera/). - -{{% /expand%}} - -## Add to a dataset - -{{< tabs >}} -{{% tab name="Web UI" %}} - -1. Open the [**DATA** page](https://app.viam.com/data/view). - -1. Navigate to the **ALL DATA** tab. - -1. Use the checkbox in the upper left of each image to select labeled images. - -1. Click the **Add to dataset** button, select a dataset, and click the **Add \ images** button to add the selected images to the dataset. - -{{% /tab %}} -{{% tab name="CLI" %}} - -Use the Viam CLI to filter images by label and add the filtered images to a dataset: - -1. First, [create a dataset](/data-ai/train/create-dataset/), if you haven't already. - -1. If you just created a dataset, use the dataset ID output by the creation command. - If your dataset already exists, run the following command to get a list of dataset names and corresponding IDs: - - ```sh {class="command-line" data-prompt="$"} - viam dataset list - ``` - -1. Run the following [command](/dev/tools/cli/#dataset) to add all images labeled with a subset of tags to the dataset, replacing the `` placeholder with the dataset ID output by the command in the previous step: - - ```sh {class="command-line" data-prompt="$"} - viam dataset data add filter --dataset-id= --tags=red_star,blue_square - ``` - -{{% /tab %}} -{{% tab name="Python" %}} - -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`data_client.add_binary_data_to_dataset_by_ids`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - -```python -# Connect to Viam client -dial_options = DialOptions( - credentials=Credentials( - type="api-key", - payload=API_KEY, - ), - auth_entity=API_KEY_ID, -) - -viam_client = await ViamClient.create_from_dial_options(dial_options) -data_client = viam_client.data_client - -# Add image to dataset -await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[EXISTING_IMAGE_ID], - dataset_id=EXISTING_DATASET_ID -) - -print("Image added to dataset successfully") -viam_client.close() -``` - -{{% /tab %}} -{{% tab name="Go" %}} - -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`DataClient.AddBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - -```go -ctx := context.Background() -viamClient, err := client.New(ctx, "", logger) -if err != nil { - log.Fatal(err) -} -defer viamClient.Close(ctx) - -dataClient := viamClient.DataClient() - -// Add image to dataset -err = dataClient.AddBinaryDataToDatasetByIDs( - context.Background(), - []string{ExistingImageID}, - ExistingDatasetID, -) -if err != nil { - return fmt.Errorf("failed to add image to dataset: %w", err) -} - -fmt.Println("Image added to dataset successfully") -``` - -{{% /tab %}} -{{% tab name="TypeScript" %}} - -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - -```typescript -const client: ViamClient = await createViamClient({ - credential: { - type: "api-key", - payload: API_KEY, - }, - authEntity: API_KEY_ID, -}); - -const dataClient = client.dataClient; - -// Add image to dataset -await dataClient.addBinaryDataToDatasetByIds({ - binaryIds: [EXISTING_IMAGE_ID], - datasetId: EXISTING_DATASET_ID, -}); - -console.log("Image added to dataset successfully"); -client.disconnect(); -``` - -{{% /tab %}} -{{% tab name="Flutter" %}} - -To add an image to a dataset, find the binary data ID for the image and the dataset ID. -Pass both IDs to [`dataClient.addBinaryDataToDatasetByIDs`](/dev/reference/apis/data-client/#addbinarydatatodatasetbyids): - -```dart -final client = await ViamClient.withApiKey( - apiKeyId: apiKeyId, - apiKey: apiKey, -); - -final dataClient = client.dataClient; - -try { - // Add image to dataset - await dataClient.addBinaryDataToDatasetByIds( - binaryIds: [existingImageId], - datasetId: existingDatasetId, - ); - - print('Image added to dataset successfully'); -} finally { - await client.close(); -} -``` - -{{% /tab %}} -{{< /tabs >}} - -## Add all images captured by a specific machine to a dataset - -The following script adds all images captured from a certain machine to a new dataset: - -{{< tabs >}} -{{% tab name="Python" %}} - -```python -import asyncio -from typing import List, Optional - -from viam.rpc.dial import DialOptions, Credentials -from viam.app.viam_client import ViamClient -from viam.utils import create_filter - -# Configuration constants – replace with your actual values -DATASET_NAME = "" # a unique, new name for the dataset you want to create -ORG_ID = "" # your organization ID, find in your organization settings -PART_ID = "" # ID of machine that captured images, find in machine config -API_KEY = "" # API key, find or create in your organization settings -API_KEY_ID = "" # API key ID, find or create in your organization settings - -# Adjust the maximum number of images to add to the dataset -MAX_MATCHES = 500 - - -async def connect() -> ViamClient: - """Establish a connection to the Viam client using API credentials.""" - dial_options = DialOptions( - credentials=Credentials( - type="api-key", - payload=API_KEY, - ), - auth_entity=API_KEY_ID, - ) - return await ViamClient.create_from_dial_options(dial_options) - - -async def fetch_binary_data_ids(data_client, part_id: str) -> List[str]: - """Fetch binary data metadata and return a list of BinaryData objects.""" - data_filter = create_filter(part_id=part_id) - all_matches = [] - last: Optional[str] = None - - print("Getting data for part...") - - while len(all_matches) < MAX_MATCHES: - print("Fetching more data...") - data, _, last = await data_client.binary_data_by_filter( - data_filter, - limit=50, - last=last, - include_binary_data=False, - ) - if not data: - break - all_matches.extend(data) - - return all_matches - - -async def main() -> int: - """Main execution function.""" - viam_client = await connect() - data_client = viam_client.data_client - - matching_data = await fetch_binary_data_ids(data_client, PART_ID) - - print("Creating dataset...") - - try: - dataset_id = await data_client.create_dataset( - name=DATASET_NAME, - organization_id=ORG_ID, - ) - print(f"Created dataset: {dataset_id}") - except Exception as e: - print("Error creating dataset. It may already exist.") - print("See: https://app.viam.com/data/datasets") - print(f"Exception: {e}") - return 1 - - print("Adding data to dataset...") - - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[ - obj.metadata.binary_data_id for obj in matching_data - ], - dataset_id=dataset_id - ) - - print("Added files to dataset.") - print( - f"See dataset: https://app.viam.com/data/datasets?id={dataset_id}" - ) - - viam_client.close() - return 0 - -if __name__ == "__main__": - asyncio.run(main()) -``` - -{{% /tab %}} -{{% tab name="Go" %}} - -```go -package main - -import ( - "context" - "fmt" - "log" - - "go.viam.com/rdk/rpc" - "go.viam.com/utils" - "go.viam.com/rdk/app" - "go.viam.com/rdk/app/data" -) - -// Configuration constants - replace with your actual values -const ( - DATASET_NAME = "" // a unique, new name for the dataset you want to create - ORG_ID = "" // your organization ID, find in your organization settings - PART_ID = "" // ID of machine that captured target images, find in machine config - API_KEY = "" // API key, find or create in your organization settings - API_KEY_ID = "" // API key ID, find or create in your organization settings - MAX_MATCHES = 500 -) - -func connect(ctx context.Context) (*app.ViamClient, error) { - client, err := app.NewViamClientWithAPIKey(ctx, API_KEY, API_KEY_ID) - if err != nil { - return nil, fmt.Errorf("failed to create client: %w", err) - } - return client, nil -} - -func fetchBinaryDataIDs( - ctx context.Context, - dataClient data.DataServiceClient, - partID string) ([]*data.BinaryData, error) { - filter := &data.Filter{ - PartId: partID, - } - - var allMatches []*data.BinaryData - var last string - - fmt.Println("Getting data for part...") - - for len(allMatches) < MAX_MATCHES { - fmt.Println("Fetching more data...") - - resp, err := dataClient.BinaryDataByFilter(ctx, &data.BinaryDataByFilterRequest{ - DataRequest: &data.DataRequest{ - Filter: filter, - Limit: 50, - Last: last, - IncludeBinaryData: false, - }, - }) - if err != nil { - return nil, fmt.Errorf("failed to fetch binary data: %w", err) - } - - if len(resp.Data) == 0 { - break - } - - allMatches = append(allMatches, resp.Data...) - last = resp.Last - } - - return allMatches, nil -} - -func main() { - ctx := context.Background() - - viamClient, err := connect(ctx) - if err != nil { - log.Fatalf("Connection failed: %v", err) - } - defer viamClient.Close() - - dataClient := viamClient.DataClient - - matchingData, err := fetchBinaryDataIDs(ctx, dataClient, PART_ID) - if err != nil { - log.Fatalf("Failed to fetch binary data: %v", err) - } - - fmt.Println("Creating dataset...") - - datasetResp, err := dataClient.CreateDataset(ctx, &data.CreateDatasetRequest{ - Name: DATASET_NAME, - OrganizationId: ORG_ID, - }) - if err != nil { - fmt.Println("Error creating dataset. It may already exist.") - fmt.Println("See: https://app.viam.com/data/datasets") - fmt.Printf("Exception: %v\n", err) - return - } - - datasetID := datasetResp.Id - fmt.Printf("Created dataset: %s\n", datasetID) - - fmt.Println("Adding data to dataset...") - - var binaryIDs []string - for _, obj := range matchingData { - binaryIDs = append(binaryIDs, obj.Metadata.Id) - } - - _, err = dataClient.AddBinaryDataToDatasetByIds(ctx, &data.AddBinaryDataToDatasetByIdsRequest{ - BinaryIds: binaryIDs, - DatasetId: datasetID, - }) - if err != nil { - log.Fatalf("Failed to add binary data to dataset: %v", err) - } - - fmt.Println("Added files to dataset.") - fmt.Printf("See dataset: https://app.viam.com/data/datasets?id=%s\n", datasetID) -} - -``` - -{{% /tab %}} -{{% tab name="TypeScript" %}} - -```typescript -import { ViamClient, createViamClient } from "@viamrobotics/sdk"; -import { DataServiceClient } from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb_service"; -import { - BinaryDataByFilterRequest, - CreateDatasetRequest, - AddBinaryDataToDatasetByIdsRequest, - Filter, - DataRequest, -} from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb"; - -// Configuration constants - replace with your actual values -const DATASET_NAME = ""; // a unique, new name for the dataset you want to create -const ORG_ID = ""; // your organization ID, find in your organization settings -const PART_ID = ""; // ID of machine that captured target images, find in machine config -const API_KEY = ""; // API key, find or create in your organization settings -const API_KEY_ID = ""; // API key ID, find or create in your organization settings -const MAX_MATCHES = 500; - -async function connect(): Promise { - return await createViamClient({ - credential: { - type: "api-key", - authEntity: API_KEY_ID, - payload: API_KEY, - }, - }); -} - -async function fetchBinaryDataIds( - dataClient: DataServiceClient, - partId: string, -): Promise { - const filter = new Filter(); - filter.setPartId(partId); - - const allMatches: any[] = []; - let last = ""; - - console.log("Getting data for part..."); - - while (allMatches.length < MAX_MATCHES) { - console.log("Fetching more data..."); - - const dataRequest = new DataRequest(); - dataRequest.setFilter(filter); - dataRequest.setLimit(50); - dataRequest.setLast(last); - dataRequest.setIncludeBinaryData(false); - - const request = new BinaryDataByFilterRequest(); - request.setDataRequest(dataRequest); - - const response = await dataClient.binaryDataByFilter(request); - const data = response.getDataList(); - - if (data.length === 0) { - break; - } - - allMatches.push(...data); - last = response.getLast(); - } - - return allMatches; -} - -async function main(): Promise { - const viamClient = await connect(); - const dataClient = viamClient.dataClient; - - const matchingData = await fetchBinaryDataIds(dataClient, PART_ID); - - console.log("Creating dataset..."); - - try { - const createRequest = new CreateDatasetRequest(); - createRequest.setName(DATASET_NAME); - createRequest.setOrganizationId(ORG_ID); - - const datasetResponse = await dataClient.createDataset(createRequest); - const datasetId = datasetResponse.getId(); - console.log(`Created dataset: ${datasetId}`); - - console.log("Adding data to dataset..."); - - const binaryIds = matchingData.map( - (obj) => obj.getMetadata()?.getId() || "", - ); - - const addRequest = new AddBinaryDataToDatasetByIdsRequest(); - addRequest.setBinaryIdsList(binaryIds); - addRequest.setDatasetId(datasetId); - - await dataClient.addBinaryDataToDatasetByIds(addRequest); - - console.log("Added files to dataset."); - console.log( - `See dataset: https://app.viam.com/data/datasets?id=${datasetId}`, - ); - - viamClient.disconnect(); - return 0; - } catch (error) { - console.log("Error creating dataset. It may already exist."); - console.log("See: https://app.viam.com/data/datasets"); - console.log(`Exception: ${error}`); - viamClient.disconnect(); - return 1; - } -} - -if (require.main === module) { - main().then((code) => process.exit(code)); -} -``` - -{{% /tab %}} -{{% tab name="Flutter" %}} - -```dart -import 'package:viam_sdk/viam_sdk.dart'; - -// Configuration constants - replace with your actual values -const String datasetName = ""; // a unique, new name for the dataset you want to create -const String orgId = ""; // your organization ID, find in your organization settings -const String partId = ""; // ID of machine that captured target images, find in machine config -const String apiKey = ""; // API key, find or create in your organization settings -const String apiKeyId = ""; // API key ID, find or create in your organization settings -const int maxMatches = 500; - -Future connect() async { - return await ViamClient.withApiKey( - apiKey: apiKey, - apiKeyId: apiKeyId, - ); -} - -Future> fetchBinaryDataIds( - DataClient dataClient, String partId) async { - final filter = Filter(partId: partId); - final List allMatches = []; - String? last; - - print("Getting data for part..."); - - while (allMatches.length < maxMatches) { - print("Fetching more data..."); - - final response = await dataClient.binaryDataByFilter( - filter: filter, - limit: 50, - last: last, - includeBinaryData: false, - ); - - if (response.data.isEmpty) { - break; - } - - allMatches.addAll(response.data); - last = response.last; - } - - return allMatches; -} - -Future main() async { - final viamClient = await connect(); - final dataClient = viamClient.dataClient; - - final matchingData = await fetchBinaryDataIds(dataClient, partId); - - print("Creating dataset..."); - - try { - final datasetId = await dataClient.createDataset( - name: datasetName, - organizationId: orgId, - ); - print("Created dataset: $datasetId"); - - print("Adding data to dataset..."); - - final binaryIds = matchingData - .map((obj) => obj.metadata.binaryDataId) - .toList(); - - await dataClient.addBinaryDataToDatasetByIds( - binaryIds: binaryIds, - datasetId: datasetId, - ); - - print("Added files to dataset."); - print("See dataset: https://app.viam.com/data/datasets?id=$datasetId"); - - viamClient.close(); - return 0; - - } catch (error) { - print("Error creating dataset. It may already exist."); - print("See: https://app.viam.com/data/datasets"); - print("Exception: $error"); - viamClient.close(); - return 1; - } -} -``` - -{{% /tab %}} -{{< /tabs >}} - -## Capture, annotate, and add images to a dataset - -The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. -You can use this logic to expand and improve your datasets continuously over time as your application runs. -Re-train your ML model on the improved dataset to improve the ML model. - -{{< tabs >}} -{{% tab name="Python" %}} - -```python -import asyncio -import os -import time -from typing import Optional -from io import BytesIO - -from PIL import Image -from viam.app.app_client import AppClient -from viam.logging import getLogger - -# Global machine configuration -MACHINE_PART_ID = "your-machine-part-id-here" - - -class DataCollector: - def __init__(self, - component_name: str, dataset_id: str, - api_key_id: str, api_key: str): - - self.logger = getLogger(__name__) - self.component_name = component_name - self.dataset_id = dataset_id - self.api_key_id = api_key_id - self.api_key = api_key - - async def capture_and_store_image(self, - processed_image: Image.Image, - classification: str) -> None: - - if not MACHINE_PART_ID: - raise ValueError("machine part ID not configured") - - # Create fresh data client connection - async with await self._create_data_client() as data_client: - image_data = self._encode_image_to_png(processed_image) - - # Generate unique filename with timestamp - timestamp = int(time.time()) - filename = f"{classification}-sample-{timestamp}.png" - - component_type = "rdk:component:camera" - - upload_metadata = { - "component_type": component_type, - "component_name": self.component_name, - "file_name": filename, - "file_extension": "png" - } - - try: - file_id = await data_client.file_upload_from_bytes( - part_id=MACHINE_PART_ID, - data=image_data, - **upload_metadata - ) - - # Associate file with dataset immediately - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[file_id], - dataset_id=self.dataset_id - ) - - self.logger.info( - f"successfully added {classification} image to dataset " - f"{self.dataset_id} (file ID: {file_id}, " - f"machine: {MACHINE_PART_ID})" - ) - - except Exception as e: - self.logger.error(f"failed to upload and associate image: {e}") - raise - - async def _create_data_client(self) -> AppClient: - if not self.api_key_id or not self.api_key: - raise ValueError("API credentials not configured") - - try: - client = AppClient.create_from_dial_options( - dial_options={ - "auth_entity": self.api_key_id, - "credentials": { - "type": "api-key", - "payload": self.api_key - } - } - ) - return client - - except Exception as e: - raise ValueError(f"failed to create app client: {e}") - - def _encode_image_to_png(self, img: Image.Image) -> bytes: - buffer = BytesIO() - img.save(buffer, format='PNG', optimize=True) - return buffer.getvalue() - - -def create_data_collector( - component_name: str, - dataset_id: str, - api_key_id: str, - api_key: str - ) -> DataCollector: - if not component_name: - raise ValueError("component name is required") - if not dataset_id: - raise ValueError("dataset ID is required") - if not api_key_id or not api_key: - raise ValueError("API credentials are required") - - return DataCollector( - component_name=component_name, - dataset_id=dataset_id, - api_key_id=api_key_id, - api_key=api_key - ) - - -# Example usage -async def main(): - collector = create_data_collector( - component_name="main_camera", - dataset_id="your-dataset-id", - api_key_id="your-api-key-id", - api_key="your-api-key" - ) - - # Example with PIL Image - sample_image = Image.new('RGB', (640, 480), color='red') - await collector.capture_and_store_image(sample_image, "positive_sample") - -if __name__ == "__main__": - asyncio.run(main()) -``` - -{{% /tab %}} -{{% tab name="Go" %}} - -```go -package datasetcreation - -import ( - "context" - "fmt" - "image" - "time" - - "go.viam.com/rdk/logging" - "go.viam.com/rdk/services/datamanager/app" - "go.viam.com/rdk/app/appclient" -) - -var MachinePartID string = "your-machine-part-id-here" - -type DataCollector struct { - logger logging.Logger - componentName string - datasetID string - apiKeyID string - apiKey string -} - -func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { - if MachinePartID == "" { - return fmt.Errorf("machine part ID not configured") - } - - // Create fresh data client connection - dataClient, err := dc.createDataClient(ctx) - if err != nil { - dc.logger.Errorf("failed to create data client: %v", err) - return fmt.Errorf("data client initialization failed: %w", err) - } - defer dataClient.Close() - - imageData, err := dc.encodeImageToPNG(processedImage) - if err != nil { - dc.logger.Errorf("image encoding failed: %v", err) - return fmt.Errorf("failed to encode image: %w", err) - } - - // Generate unique filename with timestamp - timestamp := time.Now().Unix() - filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) - - componentType := "rdk:component:camera" - fileExtension := "png" - - uploadOptions := app.FileUploadOptions{ - ComponentType: &componentType, - ComponentName: &dc.componentName, - FileName: &filename, - FileExtension: &fileExtension, - } - - fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) - if err != nil { - dc.logger.Errorf("file upload failed for %s: %v", filename, err) - return fmt.Errorf("failed to upload image: %w", err) - } - - // Associate file with dataset immediately - err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) - if err != nil { - dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) - return fmt.Errorf("failed to add image to dataset: %w", err) - } - - dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", - classification, dc.datasetID, fileID, MachinePartID) - - return nil -} - -func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { - if dc.apiKeyID == "" || dc.apiKey == "" { - return nil, fmt.Errorf("API credentials not configured") - } - - client, err := appclient.New(ctx, appclient.Config{ - KeyID: dc.apiKeyID, - Key: dc.apiKey, - }) - if err != nil { - return nil, fmt.Errorf("failed to create app client: %w", err) - } - - return client.DataClient, nil -} - -func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { - // PNG encoding implementation - return nil, nil // Placeholder -} - -func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { - if logger == nil { - return nil, fmt.Errorf("logger is required") - } - if componentName == "" { - return nil, fmt.Errorf("component name is required") - } - if datasetID == "" { - return nil, fmt.Errorf("dataset ID is required") - } - if apiKeyID == "" || apiKey == "" { - return nil, fmt.Errorf("API credentials are required") - } - - return &DataCollector{ - logger: logger, - componentName: componentName, - datasetID: datasetID, - apiKeyID: apiKeyID, - apiKey: apiKey, - }, nil -} -``` - -{{% /tab %}} -{{% tab name="TypeScript" %}} - -```typescript -import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; -import { AppClient, DataClient } from "@viamrobotics/app-client"; -import { Logger } from "@viamrobotics/utils"; - -const MACHINE_PART_ID: string = "your-machine-part-id-here"; - -interface FileUploadOptions { - componentType?: string; - componentName?: string; - fileName?: string; - fileExtension?: string; -} - -export class DataCollector { - private logger: Logger; - private componentName: string; - private datasetId: string; - private apiKeyId: string; - private apiKey: string; - - constructor( - logger: Logger, - componentName: string, - datasetId: string, - apiKeyId: string, - apiKey: string, - ) { - this.logger = logger; - this.componentName = componentName; - this.datasetId = datasetId; - this.apiKeyId = apiKeyId; - this.apiKey = apiKey; - } - - async captureAndStoreImage( - processedImage: ArrayBuffer, - classification: string, - ): Promise { - if (!MACHINE_PART_ID) { - throw new Error("Machine part ID not configured"); - } - - let dataClient: DataClient | null = null; - - try { - dataClient = await this.createDataClient(); - - const imageData = await this.encodeImageToPng(processedImage); - const timestamp = Math.floor(Date.now() / 1000); - const filename = `${classification}-sample-${timestamp}.png`; - - const componentType = "rdk:component:camera"; - const fileExtension = "png"; - - const uploadOptions: FileUploadOptions = { - componentType, - componentName: this.componentName, - fileName: filename, - fileExtension, - }; - - const fileId = await dataClient.fileUploadFromBytes( - MACHINE_PART_ID, - new Uint8Array(imageData), - uploadOptions, - ); - - await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); - - this.logger.info( - `Successfully added ${classification} image to dataset ${this.datasetId} ` + - `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, - ); - } catch (error) { - this.logger.error(`File upload failed for ${classification}: ${error}`); - throw new Error(`Failed to upload image: ${error}`); - } finally { - if (dataClient) { - await dataClient.close(); - } - } - } - - private async createDataClient(): Promise { - if (!this.apiKeyId || !this.apiKey) { - throw new Error("API credentials not configured"); - } - - const appClient = new AppClient({ - apiKeyId: this.apiKeyId, - apiKey: this.apiKey, - }); - - return appClient.dataClient(); - } - - private async encodeImageToPng(image: ArrayBuffer): Promise { - try { - // PNG encoding implementation would depend on your image processing library - // This is a placeholder - you would use a library like 'pngjs' or 'canvas' - return image; // Assuming image is already PNG encoded - } catch (error) { - this.logger.error(`Image encoding failed: ${error}`); - throw new Error(`Failed to encode image: ${error}`); - } - } - - static create( - logger: Logger, - componentName: string, - datasetId: string, - apiKeyId: string, - apiKey: string, - ): DataCollector { - if (!logger) { - throw new Error("Logger is required"); - } - if (!componentName) { - throw new Error("Component name is required"); - } - if (!datasetId) { - throw new Error("Dataset ID is required"); - } - if (!apiKeyId || !apiKey) { - throw new Error("API credentials are required"); - } - - return new DataCollector( - logger, - componentName, - datasetId, - apiKeyId, - apiKey, - ); - } -} -``` - -{{% /tab %}} -{{% tab name="Flutter" %}} - -```dart -import 'dart:typed_data'; -import 'dart:io'; -import 'package:viam_sdk/viam_sdk.dart'; -import 'package:viam_sdk/src/app/app_client.dart'; -import 'package:viam_sdk/src/app/data_client.dart'; -import 'package:image/image.dart' as img; - -const String machinePartId = 'your-machine-part-id-here'; - -class DataCollector { - final Logging logger; - final String componentName; - final String datasetId; - final String apiKeyId; - final String apiKey; - - DataCollector({ - required this.logger, - required this.componentName, - required this.datasetId, - required this.apiKeyId, - required this.apiKey, - }); - - Future captureAndStoreImage( - Image processedImage, - String classification, - ) async { - if (machinePartId.isEmpty) { - throw Exception('Machine part ID not configured'); - } - - DataClient? dataClient; - try { - dataClient = await _createDataClient(); - - final imageData = await _encodeImageToPng(processedImage); - final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; - final filename = '$classification-sample-$timestamp.png'; - - const componentType = 'rdk:component:camera'; - const fileExtension = 'png'; - - final uploadOptions = FileUploadOptions( - componentType: componentType, - componentName: componentName, - fileName: filename, - fileExtension: fileExtension, - ); - - final fileId = await dataClient.fileUploadFromBytes( - machinePartId, - imageData, - uploadOptions, - ); - - await dataClient.addBinaryDataToDatasetByIds( - [fileId], - datasetId, - ); - - logger.info( - 'Successfully added $classification image to dataset $datasetId ' - '(file ID: $fileId, machine: $machinePartId)', - ); - } catch (error) { - logger.error('File upload failed for $classification: $error'); - rethrow; - } finally { - await dataClient?.close(); - } - } - - Future _createDataClient() async { - if (apiKeyId.isEmpty || apiKey.isEmpty) { - throw Exception('API credentials not configured'); - } - - final appClient = await AppClient.withApiKey( - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - - return appClient.dataClient; - } - - Future _encodeImageToPng(Image image) async { - try { - final pngBytes = img.encodePng(image); - return Uint8List.fromList(pngBytes); - } catch (error) { - logger.error('Image encoding failed: $error'); - throw Exception('Failed to encode image: $error'); - } - } - - static DataCollector create({ - required Logging logger, - required String componentName, - required String datasetId, - required String apiKeyId, - required String apiKey, - }) { - if (componentName.isEmpty) { - throw ArgumentError('Component name is required'); - } - if (datasetId.isEmpty) { - throw ArgumentError('Dataset ID is required'); - } - if (apiKeyId.isEmpty || apiKey.isEmpty) { - throw ArgumentError('API credentials are required'); - } - - return DataCollector( - logger: logger, - componentName: componentName, - datasetId: datasetId, - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - } -} -``` - -{{% /tab %}} -{{< /tabs >}} diff --git a/netlify.toml b/netlify.toml index a7c943cc6c..f847955ebd 100644 --- a/netlify.toml +++ b/netlify.toml @@ -57,7 +57,7 @@ [plugins.inputs] # change this key to a new one any time you need to restart from scratch - cacheKey = ["June162025"] + cacheKey = ["June302025"] # either "warn" or "error" failBuildOnError = true From dfceaf7ba220fd4e4163e02a45ec5ec07bdde704 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Mon, 30 Jun 2025 15:18:32 -0400 Subject: [PATCH 20/29] Fix consecutive blank lines --- docs/data-ai/train/capture-annotate-images.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index 0abf25e561..8d0036acf2 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -282,7 +282,6 @@ Once you have enough images, consider disabling data capture to [avoid incurring {{< /alert >}} - You can either manually add annotations through the Viam web UI, or add annotations with an existing ML model. ## Annotate images @@ -735,4 +734,3 @@ for (final detection in detections) { {{< /tabs >}} Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. - From c6ff8ae6d0f4b36d3663f08a16e869450180456b Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 09:04:38 -0400 Subject: [PATCH 21/29] Apply suggestions from code review Co-authored-by: Jessamy Taylor <75634662+JessamyT@users.noreply.github.com> --- docs/data-ai/train/create-dataset.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index be57fdd02c..217e42b2b2 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -308,7 +308,6 @@ try { ## Add all images captured by a specific machine to a dataset -The following script adds all images captured from a certain machine to a new dataset: {{< tabs >}} {{% tab name="Web UI" %}} @@ -332,6 +331,8 @@ This will select both images as well as the entire range of images between those {{% /tab %}} {{% tab name="Python" %}} +The following script adds all images captured from a certain machine to a new dataset: + ```python import asyncio from typing import List, Optional @@ -431,6 +432,8 @@ if __name__ == "__main__": {{% /tab %}} {{% tab name="Go" %}} +The following script adds all images captured from a certain machine to a new dataset: + ```go package main @@ -558,6 +561,8 @@ func main() { {{% /tab %}} {{% tab name="TypeScript" %}} +The following script adds all images captured from a certain machine to a new dataset: + ```typescript import { ViamClient, createViamClient } from "@viamrobotics/sdk"; import { DataServiceClient } from "@viamrobotics/sdk/dist/gen/app/data/v1/data_pb_service"; @@ -678,6 +683,8 @@ if (require.main === module) { {{% /tab %}} {{% tab name="Flutter" %}} +The following script adds all images captured from a certain machine to a new dataset: + ```dart import 'package:viam_sdk/viam_sdk.dart'; @@ -773,8 +780,8 @@ Future main() async { ## Capture, annotate, and add images to a dataset The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. -You can use this logic to expand and improve your datasets continuously over time as your application runs. -Re-train your ML model on the improved dataset to improve the ML model. +You can use this logic to expand and improve your datasets continuously over time. +Check the annotation accuracy in the **DATA** tab, then re-train your ML model on the improved dataset to improve the ML model. {{< tabs >}} {{% tab name="Python" %}} From 9a222f8b800fced989ca1544469eda6170749840 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 09:23:29 -0400 Subject: [PATCH 22/29] Update docs/data-ai/train/create-dataset.md --- docs/data-ai/train/create-dataset.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 217e42b2b2..fe41aaf402 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -307,8 +307,6 @@ try { {{< /tabs >}} ## Add all images captured by a specific machine to a dataset - - {{< tabs >}} {{% tab name="Web UI" %}} From 046d6edbb37e4ed9d6274293068ba5fa5bbdeaf1 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 09:31:00 -0400 Subject: [PATCH 23/29] Move annotate single step to annotate page --- docs/data-ai/train/capture-annotate-images.md | 543 ++++++++++++++++++ docs/data-ai/train/create-dataset.md | 542 ----------------- 2 files changed, 543 insertions(+), 542 deletions(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index 8d0036acf2..ffe5a23fe6 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -734,3 +734,546 @@ for (final detection in detections) { {{< /tabs >}} Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. + + +## Combine these steps: capture, annotate, and add images to a dataset in a single script + +The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. +You can use this logic to expand and improve your datasets continuously over time. +Check the annotation accuracy in the **DATA** tab, then re-train your ML model on the improved dataset to improve the ML model. + +{{< tabs >}} +{{% tab name="Python" %}} + +```python +import asyncio +import os +import time +from typing import Optional +from io import BytesIO + +from PIL import Image +from viam.app.app_client import AppClient +from viam.logging import getLogger + +# Global machine configuration +MACHINE_PART_ID = "your-machine-part-id-here" + + +class DataCollector: + def __init__(self, + component_name: str, dataset_id: str, + api_key_id: str, api_key: str): + + self.logger = getLogger(__name__) + self.component_name = component_name + self.dataset_id = dataset_id + self.api_key_id = api_key_id + self.api_key = api_key + + async def capture_and_store_image(self, + processed_image: Image.Image, + classification: str) -> None: + + if not MACHINE_PART_ID: + raise ValueError("machine part ID not configured") + + # Create fresh data client connection + async with await self._create_data_client() as data_client: + image_data = self._encode_image_to_png(processed_image) + + # Generate unique filename with timestamp + timestamp = int(time.time()) + filename = f"{classification}-sample-{timestamp}.png" + + component_type = "rdk:component:camera" + + upload_metadata = { + "component_type": component_type, + "component_name": self.component_name, + "file_name": filename, + "file_extension": "png" + } + + try: + file_id = await data_client.file_upload_from_bytes( + part_id=MACHINE_PART_ID, + data=image_data, + **upload_metadata + ) + + # Associate file with dataset immediately + await data_client.add_binary_data_to_dataset_by_ids( + binary_ids=[file_id], + dataset_id=self.dataset_id + ) + + self.logger.info( + f"successfully added {classification} image to dataset " + f"{self.dataset_id} (file ID: {file_id}, " + f"machine: {MACHINE_PART_ID})" + ) + + except Exception as e: + self.logger.error(f"failed to upload and associate image: {e}") + raise + + async def _create_data_client(self) -> AppClient: + if not self.api_key_id or not self.api_key: + raise ValueError("API credentials not configured") + + try: + client = AppClient.create_from_dial_options( + dial_options={ + "auth_entity": self.api_key_id, + "credentials": { + "type": "api-key", + "payload": self.api_key + } + } + ) + return client + + except Exception as e: + raise ValueError(f"failed to create app client: {e}") + + def _encode_image_to_png(self, img: Image.Image) -> bytes: + buffer = BytesIO() + img.save(buffer, format='PNG', optimize=True) + return buffer.getvalue() + + +def create_data_collector( + component_name: str, + dataset_id: str, + api_key_id: str, + api_key: str + ) -> DataCollector: + if not component_name: + raise ValueError("component name is required") + if not dataset_id: + raise ValueError("dataset ID is required") + if not api_key_id or not api_key: + raise ValueError("API credentials are required") + + return DataCollector( + component_name=component_name, + dataset_id=dataset_id, + api_key_id=api_key_id, + api_key=api_key + ) + + +# Example usage +async def main(): + collector = create_data_collector( + component_name="main_camera", + dataset_id="your-dataset-id", + api_key_id="your-api-key-id", + api_key="your-api-key" + ) + + # Example with PIL Image + sample_image = Image.new('RGB', (640, 480), color='red') + await collector.capture_and_store_image(sample_image, "positive_sample") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +{{% /tab %}} +{{% tab name="Go" %}} + +```go +package datasetcreation + +import ( + "context" + "fmt" + "image" + "time" + + "go.viam.com/rdk/logging" + "go.viam.com/rdk/services/datamanager/app" + "go.viam.com/rdk/app/appclient" +) + +var MachinePartID string = "your-machine-part-id-here" + +type DataCollector struct { + logger logging.Logger + componentName string + datasetID string + apiKeyID string + apiKey string +} + +func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { + if MachinePartID == "" { + return fmt.Errorf("machine part ID not configured") + } + + // Create fresh data client connection + dataClient, err := dc.createDataClient(ctx) + if err != nil { + dc.logger.Errorf("failed to create data client: %v", err) + return fmt.Errorf("data client initialization failed: %w", err) + } + defer dataClient.Close() + + imageData, err := dc.encodeImageToPNG(processedImage) + if err != nil { + dc.logger.Errorf("image encoding failed: %v", err) + return fmt.Errorf("failed to encode image: %w", err) + } + + // Generate unique filename with timestamp + timestamp := time.Now().Unix() + filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) + + componentType := "rdk:component:camera" + fileExtension := "png" + + uploadOptions := app.FileUploadOptions{ + ComponentType: &componentType, + ComponentName: &dc.componentName, + FileName: &filename, + FileExtension: &fileExtension, + } + + fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) + if err != nil { + dc.logger.Errorf("file upload failed for %s: %v", filename, err) + return fmt.Errorf("failed to upload image: %w", err) + } + + // Associate file with dataset immediately + err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) + if err != nil { + dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) + return fmt.Errorf("failed to add image to dataset: %w", err) + } + + dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", + classification, dc.datasetID, fileID, MachinePartID) + + return nil +} + +func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { + if dc.apiKeyID == "" || dc.apiKey == "" { + return nil, fmt.Errorf("API credentials not configured") + } + + client, err := appclient.New(ctx, appclient.Config{ + KeyID: dc.apiKeyID, + Key: dc.apiKey, + }) + if err != nil { + return nil, fmt.Errorf("failed to create app client: %w", err) + } + + return client.DataClient, nil +} + +func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { + // PNG encoding implementation + return nil, nil // Placeholder +} + +func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { + if logger == nil { + return nil, fmt.Errorf("logger is required") + } + if componentName == "" { + return nil, fmt.Errorf("component name is required") + } + if datasetID == "" { + return nil, fmt.Errorf("dataset ID is required") + } + if apiKeyID == "" || apiKey == "" { + return nil, fmt.Errorf("API credentials are required") + } + + return &DataCollector{ + logger: logger, + componentName: componentName, + datasetID: datasetID, + apiKeyID: apiKeyID, + apiKey: apiKey, + }, nil +} +``` + +{{% /tab %}} +{{% tab name="TypeScript" %}} + +```typescript +import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; +import { AppClient, DataClient } from "@viamrobotics/app-client"; +import { Logger } from "@viamrobotics/utils"; + +const MACHINE_PART_ID: string = "your-machine-part-id-here"; + +interface FileUploadOptions { + componentType?: string; + componentName?: string; + fileName?: string; + fileExtension?: string; +} + +export class DataCollector { + private logger: Logger; + private componentName: string; + private datasetId: string; + private apiKeyId: string; + private apiKey: string; + + constructor( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string, + ) { + this.logger = logger; + this.componentName = componentName; + this.datasetId = datasetId; + this.apiKeyId = apiKeyId; + this.apiKey = apiKey; + } + + async captureAndStoreImage( + processedImage: ArrayBuffer, + classification: string, + ): Promise { + if (!MACHINE_PART_ID) { + throw new Error("Machine part ID not configured"); + } + + let dataClient: DataClient | null = null; + + try { + dataClient = await this.createDataClient(); + + const imageData = await this.encodeImageToPng(processedImage); + const timestamp = Math.floor(Date.now() / 1000); + const filename = `${classification}-sample-${timestamp}.png`; + + const componentType = "rdk:component:camera"; + const fileExtension = "png"; + + const uploadOptions: FileUploadOptions = { + componentType, + componentName: this.componentName, + fileName: filename, + fileExtension, + }; + + const fileId = await dataClient.fileUploadFromBytes( + MACHINE_PART_ID, + new Uint8Array(imageData), + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); + + this.logger.info( + `Successfully added ${classification} image to dataset ${this.datasetId} ` + + `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, + ); + } catch (error) { + this.logger.error(`File upload failed for ${classification}: ${error}`); + throw new Error(`Failed to upload image: ${error}`); + } finally { + if (dataClient) { + await dataClient.close(); + } + } + } + + private async createDataClient(): Promise { + if (!this.apiKeyId || !this.apiKey) { + throw new Error("API credentials not configured"); + } + + const appClient = new AppClient({ + apiKeyId: this.apiKeyId, + apiKey: this.apiKey, + }); + + return appClient.dataClient(); + } + + private async encodeImageToPng(image: ArrayBuffer): Promise { + try { + // PNG encoding implementation would depend on your image processing library + // This is a placeholder - you would use a library like 'pngjs' or 'canvas' + return image; // Assuming image is already PNG encoded + } catch (error) { + this.logger.error(`Image encoding failed: ${error}`); + throw new Error(`Failed to encode image: ${error}`); + } + } + + static create( + logger: Logger, + componentName: string, + datasetId: string, + apiKeyId: string, + apiKey: string, + ): DataCollector { + if (!logger) { + throw new Error("Logger is required"); + } + if (!componentName) { + throw new Error("Component name is required"); + } + if (!datasetId) { + throw new Error("Dataset ID is required"); + } + if (!apiKeyId || !apiKey) { + throw new Error("API credentials are required"); + } + + return new DataCollector( + logger, + componentName, + datasetId, + apiKeyId, + apiKey, + ); + } +} +``` + +{{% /tab %}} +{{% tab name="Flutter" %}} + +```dart +import 'dart:typed_data'; +import 'dart:io'; +import 'package:viam_sdk/viam_sdk.dart'; +import 'package:viam_sdk/src/app/app_client.dart'; +import 'package:viam_sdk/src/app/data_client.dart'; +import 'package:image/image.dart' as img; + +const String machinePartId = 'your-machine-part-id-here'; + +class DataCollector { + final Logging logger; + final String componentName; + final String datasetId; + final String apiKeyId; + final String apiKey; + + DataCollector({ + required this.logger, + required this.componentName, + required this.datasetId, + required this.apiKeyId, + required this.apiKey, + }); + + Future captureAndStoreImage( + Image processedImage, + String classification, + ) async { + if (machinePartId.isEmpty) { + throw Exception('Machine part ID not configured'); + } + + DataClient? dataClient; + try { + dataClient = await _createDataClient(); + + final imageData = await _encodeImageToPng(processedImage); + final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; + final filename = '$classification-sample-$timestamp.png'; + + const componentType = 'rdk:component:camera'; + const fileExtension = 'png'; + + final uploadOptions = FileUploadOptions( + componentType: componentType, + componentName: componentName, + fileName: filename, + fileExtension: fileExtension, + ); + + final fileId = await dataClient.fileUploadFromBytes( + machinePartId, + imageData, + uploadOptions, + ); + + await dataClient.addBinaryDataToDatasetByIds( + [fileId], + datasetId, + ); + + logger.info( + 'Successfully added $classification image to dataset $datasetId ' + '(file ID: $fileId, machine: $machinePartId)', + ); + } catch (error) { + logger.error('File upload failed for $classification: $error'); + rethrow; + } finally { + await dataClient?.close(); + } + } + + Future _createDataClient() async { + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw Exception('API credentials not configured'); + } + + final appClient = await AppClient.withApiKey( + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + + return appClient.dataClient; + } + + Future _encodeImageToPng(Image image) async { + try { + final pngBytes = img.encodePng(image); + return Uint8List.fromList(pngBytes); + } catch (error) { + logger.error('Image encoding failed: $error'); + throw Exception('Failed to encode image: $error'); + } + } + + static DataCollector create({ + required Logging logger, + required String componentName, + required String datasetId, + required String apiKeyId, + required String apiKey, + }) { + if (componentName.isEmpty) { + throw ArgumentError('Component name is required'); + } + if (datasetId.isEmpty) { + throw ArgumentError('Dataset ID is required'); + } + if (apiKeyId.isEmpty || apiKey.isEmpty) { + throw ArgumentError('API credentials are required'); + } + + return DataCollector( + logger: logger, + componentName: componentName, + datasetId: datasetId, + apiKeyId: apiKeyId, + apiKey: apiKey, + ); + } +} +``` + +{{% /tab %}} +{{< /tabs >}} diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index fe41aaf402..217ae5c660 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -774,545 +774,3 @@ Future main() async { {{% /tab %}} {{< /tabs >}} - -## Capture, annotate, and add images to a dataset - -The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. -You can use this logic to expand and improve your datasets continuously over time. -Check the annotation accuracy in the **DATA** tab, then re-train your ML model on the improved dataset to improve the ML model. - -{{< tabs >}} -{{% tab name="Python" %}} - -```python -import asyncio -import os -import time -from typing import Optional -from io import BytesIO - -from PIL import Image -from viam.app.app_client import AppClient -from viam.logging import getLogger - -# Global machine configuration -MACHINE_PART_ID = "your-machine-part-id-here" - - -class DataCollector: - def __init__(self, - component_name: str, dataset_id: str, - api_key_id: str, api_key: str): - - self.logger = getLogger(__name__) - self.component_name = component_name - self.dataset_id = dataset_id - self.api_key_id = api_key_id - self.api_key = api_key - - async def capture_and_store_image(self, - processed_image: Image.Image, - classification: str) -> None: - - if not MACHINE_PART_ID: - raise ValueError("machine part ID not configured") - - # Create fresh data client connection - async with await self._create_data_client() as data_client: - image_data = self._encode_image_to_png(processed_image) - - # Generate unique filename with timestamp - timestamp = int(time.time()) - filename = f"{classification}-sample-{timestamp}.png" - - component_type = "rdk:component:camera" - - upload_metadata = { - "component_type": component_type, - "component_name": self.component_name, - "file_name": filename, - "file_extension": "png" - } - - try: - file_id = await data_client.file_upload_from_bytes( - part_id=MACHINE_PART_ID, - data=image_data, - **upload_metadata - ) - - # Associate file with dataset immediately - await data_client.add_binary_data_to_dataset_by_ids( - binary_ids=[file_id], - dataset_id=self.dataset_id - ) - - self.logger.info( - f"successfully added {classification} image to dataset " - f"{self.dataset_id} (file ID: {file_id}, " - f"machine: {MACHINE_PART_ID})" - ) - - except Exception as e: - self.logger.error(f"failed to upload and associate image: {e}") - raise - - async def _create_data_client(self) -> AppClient: - if not self.api_key_id or not self.api_key: - raise ValueError("API credentials not configured") - - try: - client = AppClient.create_from_dial_options( - dial_options={ - "auth_entity": self.api_key_id, - "credentials": { - "type": "api-key", - "payload": self.api_key - } - } - ) - return client - - except Exception as e: - raise ValueError(f"failed to create app client: {e}") - - def _encode_image_to_png(self, img: Image.Image) -> bytes: - buffer = BytesIO() - img.save(buffer, format='PNG', optimize=True) - return buffer.getvalue() - - -def create_data_collector( - component_name: str, - dataset_id: str, - api_key_id: str, - api_key: str - ) -> DataCollector: - if not component_name: - raise ValueError("component name is required") - if not dataset_id: - raise ValueError("dataset ID is required") - if not api_key_id or not api_key: - raise ValueError("API credentials are required") - - return DataCollector( - component_name=component_name, - dataset_id=dataset_id, - api_key_id=api_key_id, - api_key=api_key - ) - - -# Example usage -async def main(): - collector = create_data_collector( - component_name="main_camera", - dataset_id="your-dataset-id", - api_key_id="your-api-key-id", - api_key="your-api-key" - ) - - # Example with PIL Image - sample_image = Image.new('RGB', (640, 480), color='red') - await collector.capture_and_store_image(sample_image, "positive_sample") - -if __name__ == "__main__": - asyncio.run(main()) -``` - -{{% /tab %}} -{{% tab name="Go" %}} - -```go -package datasetcreation - -import ( - "context" - "fmt" - "image" - "time" - - "go.viam.com/rdk/logging" - "go.viam.com/rdk/services/datamanager/app" - "go.viam.com/rdk/app/appclient" -) - -var MachinePartID string = "your-machine-part-id-here" - -type DataCollector struct { - logger logging.Logger - componentName string - datasetID string - apiKeyID string - apiKey string -} - -func (dc *DataCollector) CaptureAndStoreImage(ctx context.Context, processedImage image.Image, classification string) error { - if MachinePartID == "" { - return fmt.Errorf("machine part ID not configured") - } - - // Create fresh data client connection - dataClient, err := dc.createDataClient(ctx) - if err != nil { - dc.logger.Errorf("failed to create data client: %v", err) - return fmt.Errorf("data client initialization failed: %w", err) - } - defer dataClient.Close() - - imageData, err := dc.encodeImageToPNG(processedImage) - if err != nil { - dc.logger.Errorf("image encoding failed: %v", err) - return fmt.Errorf("failed to encode image: %w", err) - } - - // Generate unique filename with timestamp - timestamp := time.Now().Unix() - filename := fmt.Sprintf("%s-sample-%d.png", classification, timestamp) - - componentType := "rdk:component:camera" - fileExtension := "png" - - uploadOptions := app.FileUploadOptions{ - ComponentType: &componentType, - ComponentName: &dc.componentName, - FileName: &filename, - FileExtension: &fileExtension, - } - - fileID, err := dataClient.FileUploadFromBytes(ctx, MachinePartID, imageData, &uploadOptions) - if err != nil { - dc.logger.Errorf("file upload failed for %s: %v", filename, err) - return fmt.Errorf("failed to upload image: %w", err) - } - - // Associate file with dataset immediately - err = dataClient.AddBinaryDataToDatasetByIDs(ctx, []string{fileID}, dc.datasetID) - if err != nil { - dc.logger.Errorf("dataset association failed for file %s: %v", fileID, err) - return fmt.Errorf("failed to add image to dataset: %w", err) - } - - dc.logger.Infof("successfully added %s image to dataset %s (file ID: %s, machine: %s)", - classification, dc.datasetID, fileID, MachinePartID) - - return nil -} - -func (dc *DataCollector) createDataClient(ctx context.Context) (app.AppServiceClient, error) { - if dc.apiKeyID == "" || dc.apiKey == "" { - return nil, fmt.Errorf("API credentials not configured") - } - - client, err := appclient.New(ctx, appclient.Config{ - KeyID: dc.apiKeyID, - Key: dc.apiKey, - }) - if err != nil { - return nil, fmt.Errorf("failed to create app client: %w", err) - } - - return client.DataClient, nil -} - -func (dc *DataCollector) encodeImageToPNG(img image.Image) ([]byte, error) { - // PNG encoding implementation - return nil, nil // Placeholder -} - -func NewDataCollector(logger logging.Logger, componentName, datasetID, apiKeyID, apiKey string) (*DataCollector, error) { - if logger == nil { - return nil, fmt.Errorf("logger is required") - } - if componentName == "" { - return nil, fmt.Errorf("component name is required") - } - if datasetID == "" { - return nil, fmt.Errorf("dataset ID is required") - } - if apiKeyID == "" || apiKey == "" { - return nil, fmt.Errorf("API credentials are required") - } - - return &DataCollector{ - logger: logger, - componentName: componentName, - datasetID: datasetID, - apiKeyID: apiKeyID, - apiKey: apiKey, - }, nil -} -``` - -{{% /tab %}} -{{% tab name="TypeScript" %}} - -```typescript -import { createRobotClient, RobotClient } from "@viamrobotics/sdk"; -import { AppClient, DataClient } from "@viamrobotics/app-client"; -import { Logger } from "@viamrobotics/utils"; - -const MACHINE_PART_ID: string = "your-machine-part-id-here"; - -interface FileUploadOptions { - componentType?: string; - componentName?: string; - fileName?: string; - fileExtension?: string; -} - -export class DataCollector { - private logger: Logger; - private componentName: string; - private datasetId: string; - private apiKeyId: string; - private apiKey: string; - - constructor( - logger: Logger, - componentName: string, - datasetId: string, - apiKeyId: string, - apiKey: string, - ) { - this.logger = logger; - this.componentName = componentName; - this.datasetId = datasetId; - this.apiKeyId = apiKeyId; - this.apiKey = apiKey; - } - - async captureAndStoreImage( - processedImage: ArrayBuffer, - classification: string, - ): Promise { - if (!MACHINE_PART_ID) { - throw new Error("Machine part ID not configured"); - } - - let dataClient: DataClient | null = null; - - try { - dataClient = await this.createDataClient(); - - const imageData = await this.encodeImageToPng(processedImage); - const timestamp = Math.floor(Date.now() / 1000); - const filename = `${classification}-sample-${timestamp}.png`; - - const componentType = "rdk:component:camera"; - const fileExtension = "png"; - - const uploadOptions: FileUploadOptions = { - componentType, - componentName: this.componentName, - fileName: filename, - fileExtension, - }; - - const fileId = await dataClient.fileUploadFromBytes( - MACHINE_PART_ID, - new Uint8Array(imageData), - uploadOptions, - ); - - await dataClient.addBinaryDataToDatasetByIds([fileId], this.datasetId); - - this.logger.info( - `Successfully added ${classification} image to dataset ${this.datasetId} ` + - `(file ID: ${fileId}, machine: ${MACHINE_PART_ID})`, - ); - } catch (error) { - this.logger.error(`File upload failed for ${classification}: ${error}`); - throw new Error(`Failed to upload image: ${error}`); - } finally { - if (dataClient) { - await dataClient.close(); - } - } - } - - private async createDataClient(): Promise { - if (!this.apiKeyId || !this.apiKey) { - throw new Error("API credentials not configured"); - } - - const appClient = new AppClient({ - apiKeyId: this.apiKeyId, - apiKey: this.apiKey, - }); - - return appClient.dataClient(); - } - - private async encodeImageToPng(image: ArrayBuffer): Promise { - try { - // PNG encoding implementation would depend on your image processing library - // This is a placeholder - you would use a library like 'pngjs' or 'canvas' - return image; // Assuming image is already PNG encoded - } catch (error) { - this.logger.error(`Image encoding failed: ${error}`); - throw new Error(`Failed to encode image: ${error}`); - } - } - - static create( - logger: Logger, - componentName: string, - datasetId: string, - apiKeyId: string, - apiKey: string, - ): DataCollector { - if (!logger) { - throw new Error("Logger is required"); - } - if (!componentName) { - throw new Error("Component name is required"); - } - if (!datasetId) { - throw new Error("Dataset ID is required"); - } - if (!apiKeyId || !apiKey) { - throw new Error("API credentials are required"); - } - - return new DataCollector( - logger, - componentName, - datasetId, - apiKeyId, - apiKey, - ); - } -} -``` - -{{% /tab %}} -{{% tab name="Flutter" %}} - -```dart -import 'dart:typed_data'; -import 'dart:io'; -import 'package:viam_sdk/viam_sdk.dart'; -import 'package:viam_sdk/src/app/app_client.dart'; -import 'package:viam_sdk/src/app/data_client.dart'; -import 'package:image/image.dart' as img; - -const String machinePartId = 'your-machine-part-id-here'; - -class DataCollector { - final Logging logger; - final String componentName; - final String datasetId; - final String apiKeyId; - final String apiKey; - - DataCollector({ - required this.logger, - required this.componentName, - required this.datasetId, - required this.apiKeyId, - required this.apiKey, - }); - - Future captureAndStoreImage( - Image processedImage, - String classification, - ) async { - if (machinePartId.isEmpty) { - throw Exception('Machine part ID not configured'); - } - - DataClient? dataClient; - try { - dataClient = await _createDataClient(); - - final imageData = await _encodeImageToPng(processedImage); - final timestamp = DateTime.now().millisecondsSinceEpoch ~/ 1000; - final filename = '$classification-sample-$timestamp.png'; - - const componentType = 'rdk:component:camera'; - const fileExtension = 'png'; - - final uploadOptions = FileUploadOptions( - componentType: componentType, - componentName: componentName, - fileName: filename, - fileExtension: fileExtension, - ); - - final fileId = await dataClient.fileUploadFromBytes( - machinePartId, - imageData, - uploadOptions, - ); - - await dataClient.addBinaryDataToDatasetByIds( - [fileId], - datasetId, - ); - - logger.info( - 'Successfully added $classification image to dataset $datasetId ' - '(file ID: $fileId, machine: $machinePartId)', - ); - } catch (error) { - logger.error('File upload failed for $classification: $error'); - rethrow; - } finally { - await dataClient?.close(); - } - } - - Future _createDataClient() async { - if (apiKeyId.isEmpty || apiKey.isEmpty) { - throw Exception('API credentials not configured'); - } - - final appClient = await AppClient.withApiKey( - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - - return appClient.dataClient; - } - - Future _encodeImageToPng(Image image) async { - try { - final pngBytes = img.encodePng(image); - return Uint8List.fromList(pngBytes); - } catch (error) { - logger.error('Image encoding failed: $error'); - throw Exception('Failed to encode image: $error'); - } - } - - static DataCollector create({ - required Logging logger, - required String componentName, - required String datasetId, - required String apiKeyId, - required String apiKey, - }) { - if (componentName.isEmpty) { - throw ArgumentError('Component name is required'); - } - if (datasetId.isEmpty) { - throw ArgumentError('Dataset ID is required'); - } - if (apiKeyId.isEmpty || apiKey.isEmpty) { - throw ArgumentError('API credentials are required'); - } - - return DataCollector( - logger: logger, - componentName: componentName, - datasetId: datasetId, - apiKeyId: apiKeyId, - apiKey: apiKey, - ); - } -} -``` - -{{% /tab %}} -{{< /tabs >}} From b40c7137f8adbf0b4a50dd376b1fb2cf5e828bb4 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 10:50:18 -0400 Subject: [PATCH 24/29] Delete duplicate newline --- docs/data-ai/train/capture-annotate-images.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index ffe5a23fe6..721b1aad27 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -735,7 +735,6 @@ for (final detection in detections) { Once you've annotated your dataset, you can [train](/data-ai/train/train-tflite/) an ML model to make inferences. - ## Combine these steps: capture, annotate, and add images to a dataset in a single script The following example demonstrates how you can capture an image, use an ML model to generate annotations, and then add the image to a dataset. From 6450cb315f5f7101bc5fb409a1b6c73a96e4f6e9 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 10:59:29 -0400 Subject: [PATCH 25/29] Add newline --- docs/data-ai/train/create-dataset.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/data-ai/train/create-dataset.md b/docs/data-ai/train/create-dataset.md index 217ae5c660..f6143e68ca 100644 --- a/docs/data-ai/train/create-dataset.md +++ b/docs/data-ai/train/create-dataset.md @@ -307,6 +307,7 @@ try { {{< /tabs >}} ## Add all images captured by a specific machine to a dataset + {{< tabs >}} {{% tab name="Web UI" %}} From 9f3126c9be2af52d14c0d79e2220b17caaa163eb Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 12:29:58 -0400 Subject: [PATCH 26/29] Update docs/data-ai/train/capture-annotate-images.md Co-authored-by: Naomi Pentrel <5212232+npentrel@users.noreply.github.com> --- docs/data-ai/train/capture-annotate-images.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index 721b1aad27..948ceb22de 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -300,7 +300,7 @@ To create a training dataset for classification, annotate tags to describe your {{< alert title="Tip" color="tip" >}} -Unless you already have an ML model that can generate tags for your dataset, use the Web UI to annotate. +If you have an ML model, use code to speed up annoating your data, otherwise use the Web UI. {{< /alert >}} From 7e864d2431c61781d49bce79f9a076c63805af4b Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 12:39:03 -0400 Subject: [PATCH 27/29] Fix typo --- docs/data-ai/train/capture-annotate-images.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index 948ceb22de..b3c1e2b744 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -300,7 +300,7 @@ To create a training dataset for classification, annotate tags to describe your {{< alert title="Tip" color="tip" >}} -If you have an ML model, use code to speed up annoating your data, otherwise use the Web UI. +If you have an ML model, use code to speed up annotating your data, otherwise use the Web UI. {{< /alert >}} From b9d08b49251489c67aaaa02ab423f625cfde9f72 Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 12:52:53 -0400 Subject: [PATCH 28/29] Clean up tag verbiage --- docs/data-ai/train/capture-annotate-images.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index b3c1e2b744..73134aa21d 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -522,8 +522,7 @@ To create a training set for object detection, annotate bounding boxes to teach {{< alert title="Tip" color="tip" >}} -To start a new dataset with no preexisting data or model, use the Web UI to annotate tags. -If you have an ML model that can generate tags for your dataset, consider using the model in an SDK to speed up annotation. +If you have an ML model, use code to speed up annotating your data, otherwise use the Web UI. {{< /alert >}} From 1873f7ff377e94dd23ba97f295cfa3021342c4cf Mon Sep 17 00:00:00 2001 From: nathan contino Date: Tue, 1 Jul 2025 12:57:39 -0400 Subject: [PATCH 29/29] Verbiage improvement --- docs/data-ai/train/capture-annotate-images.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-ai/train/capture-annotate-images.md b/docs/data-ai/train/capture-annotate-images.md index 73134aa21d..4e9a078a7a 100644 --- a/docs/data-ai/train/capture-annotate-images.md +++ b/docs/data-ai/train/capture-annotate-images.md @@ -295,7 +295,7 @@ For example, you could use classification to answer the following questions: - the quality of manufacturing output `good` or `bad`? - what combination of toppings exists on a pizza: `pepperoni`, `sausage`, and `pepper`? or `pineapple`, `ham`, and `mushroom`? -Viam supports single and multiple label classification. +Viam supports single and multiple classification. To create a training dataset for classification, annotate tags to describe your images. {{< alert title="Tip" color="tip" >}}