Telekinesis Agentic Skill Library

Appearance

Illusion - Flying Things ​

Goal: Generate a labeled COCO instance-segmentation dataset of mechanical parts scattered in mid-air.

Level: Beginner

Time: ~10 minutes

Background ​

The examples/quickstart_flying_things.py script is the smallest example that exercises the full randomization pipeline: model counts and poses, materials, HDRI background, point and sun lights, and multiple camera views. Physics is disabled.

This tutorial uses telekinesis-illusion 0.2.6.

Complete Install telekinesis-illusion, activate that environment, and run this tutorial from the cloned telekinesis-illusion repository root so its assets/ directory is available.

Quickstart ​

Create quickstart_flying_things.py ​

Create the file in the telekinesis-illusion repository root:

python
"""Generate a "flying things"-type dataset."""

import argparse

import numpy as np

from telekinesis.illusion.core.engine import (
    BlenderEngine,
    PhysicsConfig,
    RenderConfig,
)
from telekinesis.illusion.core.scene import Scene
from telekinesis.illusion.core.synthetic_data_generator import (
    SyntheticDataGenerator,
)
from telekinesis.illusion.randomizer.randomizer import Randomizer
from telekinesis.illusion.randomizer.randomizer_node import (
    BackgroundRandomizer,
    CameraPoseRandomizer,
    LightPoseRandomizer,
    LightRandomizer,
    MaterialRandomizer,
    ObjectInstanceRandomizer,
    ObjectPoseRandomizer,
)
from telekinesis.illusion.sampler.camera_pose_sampler import shell_sampler
from telekinesis.illusion.types.camera import CameraConfig
from telekinesis.illusion.types.distribution import uniform
from telekinesis.illusion.types.light import Light
from telekinesis.illusion.types.model_asset import ModelAsset
from telekinesis.illusion.types.object import Object
from telekinesis.illusion.utils.assets import resolve_asset_dir
from telekinesis.illusion.writer.writer import CocoWriter


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--preview",
        action=argparse.BooleanOptionalAction,
        default=True,
        help="Open the generated dataset in the interactive viewer.",
    )
    parser.add_argument("--num-images", type=int, default=5)
    parser.add_argument("--output-dir", type=str, default=None)
    parser.add_argument("--resolution", type=int, default=720)
    parser.add_argument("--samples", type=int, default=None)
    parser.add_argument("--device", default="CUDA")
    parser.add_argument("--seed", type=int, default=None)
    args = parser.parse_args()

    if args.seed is not None:
        import random

        random.seed(args.seed)
        np.random.seed(args.seed)

    scene = Scene(
        camera_config=CameraConfig(
            image_width=args.resolution,
            image_height=args.resolution,
        )
    )
    assets_dir = resolve_asset_dir()

    scene.add(Light("key", "POINT", power=40.0, radius=0.05))
    scene.add(Light("sun", "SUN", power=0.2))
    scene.add(
        ModelAsset(
            assets_dir / "models/mechanical_parts/gearwheel_1.glb",
            object_name="part_1",
            uv_mapping="cube",
        )
    )
    scene.add(
        ModelAsset(
            assets_dir / "models/mechanical_parts/pipe_1.glb",
            object_name="part_2",
            uv_mapping="cube",
        )
    )

    randomizer = Randomizer()
    randomizer.add_randomizer(
        ObjectInstanceRandomizer(
            counts={"part_1": (1, 3), "part_2": (1, 1)},
            min_num_total_objects=2,
            max_num_total_objects=4,
        ),
        "instance_randomizer_objects",
    )

    def sample_pose(obj: Object) -> None:
        obj.set_location(
            np.random.uniform((-0.1, -0.1, 0.1), (0.1, 0.1, 0.1))
        )
        obj.set_rotation(
            np.random.uniform((-180, -180, -180), (180, 180, 180))
        )

    randomizer.add_randomizer(
        ObjectPoseRandomizer(
            pose_sampling_function=sample_pose,
            target_objects=["part_1", "part_2"],
        ),
        "pose_randomizer",
    )
    randomizer.add_randomizer(
        MaterialRandomizer(
            target_objects=["part_1", "part_2"],
            types=["metal"],
        ),
        "material_randomizer",
    )
    randomizer.add_randomizer(
        BackgroundRandomizer(
            categories=["indoor/industrial", "indoor/studio"]
        ),
        "background_randomizer",
    )
    randomizer.add_randomizer(
        LightRandomizer(
            ["key"],
            color=uniform((0.8, 0.85, 0.9), (1.0, 1.0, 1.0)),
            power=uniform(20.0, 60.0),
            radius=uniform(0.02, 0.08),
        ),
        "point_light_properties",
    )
    randomizer.add_randomizer(
        LightRandomizer(
            ["sun"],
            power=uniform(0.1, 0.4),
            angle=uniform(0.01, 0.1),
        ),
        "sun_properties",
    )
    randomizer.add_randomizer(
        LightPoseRandomizer(
            shell_sampler,
            target_lights=["key", "sun"],
            min_distance=0.3,
            radius_min=1.0,
            radius_max=1.5,
            elevation_min=35.0,
            elevation_max=75.0,
        ),
        "light_poses",
    )
    randomizer.add_randomizer(
        CameraPoseRandomizer(
            shell_sampler,
            number_of_views=2,
            radius_min=0.5,
            radius_max=0.7,
        ),
        "camera_pose_randomizer",
    )

    writer = CocoWriter(
        output_dir=args.output_dir,
        include_camera_metadata=True,
    )
    generator = SyntheticDataGenerator(
        scene=scene,
        randomizer=randomizer,
        writer=writer,
        engine=BlenderEngine(
            RenderConfig(device=args.device, samples=args.samples),
            PhysicsConfig(active=False),
        ),
    )
    generator.generate(num_images=args.num_images)

    if args.preview:
        from telekinesis.illusion.viewer.shard_viewer import view_coco

        view_coco(writer.get_output_dir())


if __name__ == "__main__":
    main()

Generate a small dataset ​

bash
python quickstart_flying_things.py --no-preview --device CPU --num-images 5 --resolution 128 --samples 8 --seed 17 --output-dir output/flying-things

Walkthrough ​

Command options ​

  • --num-images 5 writes exactly five camera views.
  • --resolution 128 --samples 8 keeps this first render small and fast.
  • --device CPU avoids requiring a CUDA-capable GPU.
  • --seed 17 makes the Python and NumPy random choices repeatable.
  • --output-dir output/flying-things selects the dataset directory.
  • --no-preview skips the interactive preview; use --preview to open it after generation.

Create the scene and register assets ​

python
scene = Scene(
    camera_config=CameraConfig(
        image_width=args.resolution,
        image_height=args.resolution,
    )
)
assets_dir = resolve_asset_dir()

scene.add(Light("key", "POINT", power=40.0, radius=0.05))
scene.add(Light("sun", "SUN", power=0.2))
scene.add(
    ModelAsset(
        assets_dir / "models/mechanical_parts/gearwheel_1.glb",
        object_name="part_1",
        uv_mapping="cube",
    )
)
scene.add(
    ModelAsset(
        assets_dir / "models/mechanical_parts/pipe_1.glb",
        object_name="part_2",
        uv_mapping="cube",
    )
)
  • CameraConfig uses the resolution selected on the command line.
  • resolve_asset_dir() finds the sample assets shipped with the repository.
  • The scene registers a point light, a sun light, a gearwheel, and a pipe. The object names are used by the randomizers.

Select objects and sample their poses ​

python
randomizer = Randomizer()
randomizer.add_randomizer(
    ObjectInstanceRandomizer(
        counts={"part_1": (1, 3), "part_2": (1, 1)},
        min_num_total_objects=2,
        max_num_total_objects=4,
    ),
    "instance_randomizer_objects",
)

def sample_pose(obj: Object) -> None:
    obj.set_location(
        np.random.uniform((-0.1, -0.1, 0.1), (0.1, 0.1, 0.1))
    )
    obj.set_rotation(
        np.random.uniform((-180, -180, -180), (180, 180, 180))
    )

randomizer.add_randomizer(
    ObjectPoseRandomizer(
        pose_sampling_function=sample_pose,
        target_objects=["part_1", "part_2"],
    ),
    "pose_randomizer",
)
  • Each scene contains one to three gearwheels and exactly one pipe, with a total of two to four objects.
  • sample_pose() samples positions inside a 20 cm box and rotations across the full degree range.
  • The pose randomizer applies that function to every visible part and rejects collisions.

Randomize appearance, lights, and cameras ​

python
randomizer.add_randomizer(
    MaterialRandomizer(["part_1", "part_2"], types=["metal"]),
    "material_randomizer",
)
randomizer.add_randomizer(
    BackgroundRandomizer(
        categories=["indoor/industrial", "indoor/studio"]
    ),
    "background_randomizer",
)
randomizer.add_randomizer(
    LightRandomizer(
        ["key"],
        power=uniform(20.0, 60.0),
        radius=uniform(0.02, 0.08),
    ),
    "point_light_properties",
)
randomizer.add_randomizer(
    LightPoseRandomizer(
        shell_sampler,
        target_lights=["key", "sun"],
        radius_min=1.0,
        radius_max=1.5,
        elevation_min=35.0,
        elevation_max=75.0,
    ),
    "light_poses",
)
randomizer.add_randomizer(
    CameraPoseRandomizer(
        shell_sampler,
        number_of_views=2,
        radius_min=0.5,
        radius_max=0.7,
    ),
    "camera_pose_randomizer",
)
  • Parts receive random metal materials and an industrial or studio HDRI background.
  • The key-light properties and both light poses vary between scenes.
  • Two cameras are sampled on a shell 0.5 to 0.7 metres from the visible objects.

Render and write the dataset ​

python
writer = CocoWriter(
    output_dir=args.output_dir,
    include_camera_metadata=True,
)
generator = SyntheticDataGenerator(
    scene=scene,
    randomizer=randomizer,
    writer=writer,
    engine=BlenderEngine(
        RenderConfig(device=args.device, samples=args.samples),
        PhysicsConfig(active=False),
    ),
)
generator.generate(num_images=args.num_images)

if args.preview:
    from telekinesis.illusion.viewer.shard_viewer import view_coco

    view_coco(writer.get_output_dir())
  • Physics is disabled, so objects remain at their sampled mid-air poses.
  • CocoWriter saves JPEG images, COCO instance annotations, and camera metadata.
  • Five requested images with two views per scene require three randomized scenes; the last render batch is trimmed to one view.

Result ​

After a successful run, inspect:

text
output/flying-things/
  coco_annotations.json
  images/
    000000.jpg
    ...

The terminal logs Generation complete: 5 images from 3 samples. Open the images and verify that object selection, poses, lighting, and camera position vary. Confirm that coco_annotations.json has five image records and annotations containing RLE segmentation, bounding boxes, areas, and the registered categories.

Run again with --preview to open the built-in COCO preview after generation. Choose a new --output-dir for a separate dataset because CocoWriter appends to existing output.

Next Steps ​

  • Add physics and a container in Parts in a Bin.
  • Use a YAML-driven Worker for sharding and dataset export.