Scale
Fixless Audits
Fixless Audits is a powerful feature designed to enhance the quality and accuracy of task auditing. Unlike regular audits, fixless audits allow auditors to submit their feedback, without the need to perform any fixes, making the auditing process much quicker and effective.
This type of audits can be created both automatically, via API, and manually through the LidarLite tool.
This document aims to provide a step by step guide on how to review the API generated audits, and also how to create an audit manually using the tool.
Create a Fixless Audit
Audits created with it are composed of Feedback Item (FI)s, which serve to indicate errors, comments, confirmations, or flags on tasks.
Within the Scale platform, Feedback Items can be dynamically updated and their states (accepted, rejected, disputed) modified as contributors address the feedback. This flexibility ensures that the auditing process is both thorough and responsive.
A key aspect of Fixless Audits is the Error-type Feedback Items, which are used to calculate quality scores. To ensure the most accurate grading, only the latest Fixless Audit for each task is considered. However, all historical audits for a task are accessible via the API and can be reviewed in the Scale Auditing tool.
By indicating the “Scope” of the FI auditors can defines the relevant component of the task (e.g., annotation_id, attribute, frame interval). Scope also includes an error category field.
Multiple Fixless Audits can be submitted for the same task, but only the most recent audit affects the quality score. This system ensures that the most up-to-date and accurate information is used for evaluation.
Additionally, Fixless Audits submitted via the API can be modified and adjusted in the Scale Auditing tool, allowing auditors to add, delete, or edit Feedback Items as needed.
Below is a detailed breakdown of the different types, states, and categories of Feedback Items, along with their respective fields and metadata:
Feedback Item | Type | Is Required | Options |
---|---|---|---|
Type: Type is determined at time of primitive creation, but can be changed in subsequent audits The type used will determine whether certain fields are required | String | Yes |
|
State : States can only be changed by Scale users, and can be changed multiple times | String | Yes |
|
Scope | Object | Yes | For more detail on the different type of scopes and their params, check the table below. |
Category Fields: Error Category | String | Yes, only for type “error“ | Geometry error Position error Orientation error Label error Attribute error Extraneous error Missing error |
Category Fields: Error Severity | String | Yes, only for type “error“ |
|
Description | String | No | A free-form text area where users can enter any relevant information or details. |
Metadata | Object | No |
Request
// --- /v1/audits --- //
// PUT /v1/audits
body FixlessAuditCreate
201 response FixlessAudit
// GET /v1/audits?id=xxx&task_id=yyy
200 response FixlessAudit[]
// --- /v1/audit --- //
// GET /v1/audit/:id
200 response FixlessAudit
Creation Payload
FixlessAuditCreate {
type: 'fixless';
result: 'accepted' | 'rejected';
task_id: string;
comments?: string;
// from task.response.annotations.url
target_response_url: string;
feedback_items?: FeedbackItemCreate[];
metadata?: { [key: string]: any };
}
FeedbackItemCreate {
type: 'comment' | 'error' | 'flag' | 'confirmation';
scope: FeedbackItemScope; // see DTO
category?:
| 'attribute'
| 'extraneous'
| 'geometry'
| 'label'
| 'missing'
| 'position';
severity?: 'mild' | 'standard' | 'severe';
description?: string;
metadata?: { [key: string]: any };
}
Feedback Item Scope
FeedbackItemScope =
| FeedbackItemScopeAnnotation
| FeedbackItemScopePolygon
| FeedbackItemScopePoint
| FeedbackItemScopeScene;
// Most errors should use this scope
FeedbackItemScopeAnnotation = {
type: 'annotation';
annotation_id: string;
// Defaults to the scene's primary sensor (typically the first lidar) // Typically used to identify a specific camera for 2D projections
sensor_id?: string | number;
// attribute name. defined iff scoped on an annotation attribute
attribute?: string;
interval: FeedbackItemTimestampRange;
}
// Only for missing annotation errors
FeedbackItemScopePolygon {
type: 'polygon';
vertices: [[x0, y0], [x1, y1], [x2, y2], ...];
// defined iff scoped on a 2d projection
camera_id?: string;
interval: FeedbackItemTimestampRange;
}
// Only for missing annotation errors
FeedbackItemScopePoint {
type: 'point';
coordinates: [x, y];
// defined iff scoped on a 2d projection
camera_id?: string;
interval: FeedbackItemTimestampRange;
}
FeedbackItemScopeScene {
type: 'scene';
// attribute name. defined iff scoped on a scene attribute
attribute?: string;
interval: FeedbackItemTimestampRange;
}
// INTERVAL
FeedbackItemInterval = {
type: 'frame_range';
// defaults to time since start of scene
epoch?: 'scene' | 'unix';
// frame index
start: number;
end: number;
};
Data Transfer Objects (DTO)
FixlessAudit {
id: string;
// SRNs can be used in place of id on endpoints
srn: 'srn:scale:avcv:audit:{{id}}'
type: 'fixless';
result: 'accepted' | 'rejected';
task_id: string;
comments?: string;
target_response: { url: string };
feedback_items?: FeedbackItem[];
metadata?: { [key: string]: any };
active: boolean; // whether this is the latest audit
source: 'api' | 'lidarlite' | 'classic';
created_by: string;
created_at: iso_date_string;
updated_at: iso_date_string;
}
FeedbackItem {
id: string;
type: 'comment' | 'error' | 'flag' | 'confirmation';
state: 'open' | 'accepted' | 'disputed' | 'rejected';
scope: FeedbackItemScope;
category?:
| 'attribute'
| 'extraneous'
| 'geometry'
| 'label'
| 'missing'
| 'position';
severity?: 'mild' | 'standard' | 'severe';
description?: string;
metadata?: { [key: string]: any };
// updates to the state field (rest are immutable)
updated_at: iso_date_string;
updated_by: string;
grader_output: {
error_weight?: number;
explanation?: string;
conflict: boolean;
}
}
FixlessAuditConflicts {
auditConflictStatus: 'accepted' | 'accepted-with-conflicts';
rejectedFeedbackItems: FeedbackItem[];
}
Example
import requests
from requests.auth import HTTPBasicAuth
headers = {"Accept": "application/json"}
auth = HTTPBasicAuth('{{ApiKey}}', '') # No password
response = requests.request(
"PUT",
"https://api.scale.com/v1/audits",
json={
"type": "fixless",
"result": "rejected",
"task_id": "task_id_here",
"comments": "test reject comment",
"target_response_url": "value from task.response.annotations.url",
"feedback_items": [
{
"type": "comment",
"scope": {
"type": "annotation",
"annotation_id": "abc123",
"sensor_id": "789xyz",
"interval": {
"type": "frame_range",
"start": 0,
"end": 10,
},
},
"severity": "mild",
"description": "feedback 1",
},
{
"type": "confirmation",
"scope": {
"type": "scene",
"attribute": "weather",
"interval": {
"type": "frame_range",
"start": 0,
"end": 100,
},
},
"description": "feedback 2",
},
],
"metadata": { "test": True },
},
headers=headers,
auth=auth,
)
response.status_code # 201
response.text
{
"id": "6594cfca95644b72685e76e1",
"type": "fixless",
"task_id": "task_id_here",
"task_type": "sensorfusion",
"result": "rejected",
"target_response": {
"url": "value from task.response.annotations.url"
},
"comments": "test reject comment",
"feedback_items": [
{
"id": "6594cfca95644b72685e76e5",
"type": "comment",
"state": "open",
"scope": {
"type": "annotation",
"annotation_id": "abc123",
"sensor_id": "789xyz",
"interval": { "type": "frame_range", "start": 0, "end": 10 }
},
"severity": "mild",
"description": "feedback 1",
"updated_at": "2024-01-03T03:08:58.200Z",
"updated_by": "user_id_123456"
},
{
"id": "6594cfca95644b72685e76e6",
"type": "confirmation",
"state": "open",
"scope": {
"type": "scene",
"attribute": "weather",
"interval": { "type": "frame_range", "start": 0, "end": 100 }
},
"description": "feedback 2",
"updated_at": "2024-01-03T03:08:58.201Z",
"updated_by": "user_id_123456"
}
],
"metadata": { "test": true },
"active": false,
"source": "api",
"created_by": "user_id_123456",
"created_at": "2024-01-03T03:08:58.162Z",
"updated_at": "2024-01-03T03:09:42.943Z"
}