Historical Time Series Ingestion API endpoints
Use the Historical Time Series Ingestion API endpoints to perform tasks related to historical time series ingestion. Before you run an endpoint, make sure that you authenticate the API request.
openapi: 3.0.0
components:
schemas:
About:
type: object
required:
- product
- component
- api_version
properties:
product:
description: Product name
type: string
component:
description: Component
type: string
api_version:
description: API versions
type: array
items:
type: string
ApiError:
title: ApiError
description: An error message returned by the server
type: object
required:
- code
- message
properties:
code:
type: string
message:
type: string
transient:
type: boolean
RelationMode:
description: 'available relation mode values (CHANGE, ASSERT).'
type: string
enum:
- CHANGE
- ASSERT
default: ASSERT
Lookup:
type: object
properties:
key:
description: Name of the lookup field
type: string
value:
description: Value for the lookup value
type: string
weak:
description: Is a weak lookup
type: boolean
example:
key: VMW_REF
value: 'jhfd7-8csjhd-fgw89-gsd:host-1234'
ResourceLookup:
type: object
properties:
key:
description: Name of the resource lookup field
value:
description: Value for the resource lookup value
example:
key: UUID
value: 1e2eeb94-bc71-11ea-b3de-0242ac130004
Metric:
type: object
properties:
name:
description: The name of the metric
type: string
example:
- CPU_UTIL
- MEM_UTIL
values:
description: >-
An object containing possible defined statistics (avg, min, max,
sum, stddev, count, pct5, pct10, pct25, pct50, pct75, pct90, pct95,
weight). Statistics are discarded and the only stored values are
val, duration and ts. When a statistic is requested, it is
calculated at runtime by the service.
type: object
required:
- ts
- duration
- val
properties:
ts:
type: array
description: 'Starting timestamp of the sample as string, compliant ISO 8601'
items:
type: string
format: date-time
example:
- 2019-10-10T04:00:00.000Z
- 2019-10-10T05:00:00.000Z
duration:
type: array
description: Sample validity period in seconds
items:
type: number
example:
- 3600
- 3600
val:
type: array
description: Timeseries value points
items:
type: number
example:
- 0.06
- 0.81
sval:
type: array
description: Timeseries value points
items:
type: string
example:
- cluster name
- operating system
min:
type: array
description: Timeseries min stat value points
items:
type: number
example:
- 0.06
- 0.81
max:
type: array
description: Timeseries max stat value points
items:
type: number
example:
- 0.06
- 0.81
avg:
type: array
description: Timeseries avg stat value points
items:
type: number
example:
- 0.06
- 0.81
sum:
type: array
description: Timeseries sum stat value points
items:
type: number
example:
- 0.06
- 0.81
count:
type: array
description: Timeseries count stat value points
items:
type: integer
example:
- 24
- 72
stddev:
type: array
description: Timeseries stddev stat value points
items:
type: number
example:
- 0.062
- 0.813
sumquad:
type: array
description: Timeseries sum stat value points
items:
type: number
example:
- 0.136
- 0.495
pct5:
type: array
description: Timeseries percentile 25th stat value points
items:
type: number
example:
- 0.06
- 0.81
pct10:
type: array
description: Timeseries percentile 25th stat value points
items:
type: number
example:
- 0.06
- 0.81
pct25:
type: array
description: Timeseries percentile 25th stat value points
items:
type: number
example:
- 0.06
- 0.81
pct50:
type: array
description: Timeseries percentile 25th stat value points
items:
type: number
example:
- 0.06
- 0.81
pct75:
type: array
description: Timeseries percentile 75th stat value points
items:
type: number
example:
- 0.06
- 0.81
pct90:
type: array
description: Timeseries percentile 90th stat value points
items:
type: number
example:
- 0.06
- 0.81
pct95:
type: array
description: Timeseries percentile 95th stat value points
items:
type: number
example:
- 0.06
- 0.81
weight:
type: array
description: Timeseries weight value points
items:
type: number
example:
- 12
- 15
example:
name: CPU_UTIL_USER
values:
ts:
- 2019-10-10T04:00:00.000Z
- 2019-10-10T05:00:00.000Z
- 2019-10-10T06:00:00.000Z
duration:
- 3600
- 3600
- 3600
val:
- 0.36
- 0.54
- 0.81
min:
- 0.36
- 0.54
- 0.81
max:
- 0.36
- 0.54
- 0.81
pct90:
- 0.33
- 0.5
- 0.77
pct95:
- 0.35
- 0.52
- 0.79
TimeseriesData:
type: object
required:
- entity_name
- entity_type
- lookup
properties:
entity_name:
description: The entity display name
type: string
example: R&D ESX host 1
entity_type:
description: >
The entity category and type specification.
String contains the category (sys, bd, dom) encoded with entity type
(like vhc:vmw).
Specification :
type: string
pattern: '^(sys|app|wkld)(:{1}[a-z]+){1,}$'
example: 'sys:vhc:vmw'
metadata:
type: object
properties:
insert_ts:
type: string
format: date-time
example: 2019-01-01T03:15:20.000Z
etl_id:
type: integer
example: 16
provider_id:
type: integer
example: 4
entity_catalog_id:
type: integer
example: 37
import_id:
description: >-
The identifier for the data import. It is generated by the
Loader and used to group notifications when data is stored.
type: string
format: uuid
example: eeb8b3d4-e6a7-11ea-adc1-0242ac120002
skip_entities_creation:
type: boolean
description: 'if true, no new entities are created'
default: false
remove_domain_from_lookup:
type: boolean
description: remove domain from lookup
default: false
remove_domain_from_sysnm:
type: boolean
description: remove domain from system name
default: true
exec_data_repair:
type: boolean
description: enable data repair mode
default: false
lookup:
description: >-
The array of key/value to use as look up. Lookup keys are processed
in specified order and if a certain position is producing an hit
previous keys should not be contradicted.
type: array
items:
$ref: '#/components/schemas/Lookup'
namespace:
description: The metric namespace
type: string
format: 'opt:meas | opt:meas:vn | opt:der | opt:mod | opt:ind'
example: 'opt:meas'
externally_defined_statistic:
description: >-
Externally defined statistic to which assign given time series
samples (like XST_H, XST_D, XST_W, XST_M)
type: string
example: XST_H
resources:
description: The data for the resources
type: array
items:
type: object
properties:
type:
description: 'the resource type, GLOBAL by default'
type: string
example: CPU
name:
type: string
example: cpu0
lookup:
description: >-
An optional look up value to be use to identify this resource
uniquely
type: array
items:
$ref: '#/components/schemas/ResourceLookup'
metrics:
description: The metrics of the resource
type: array
items:
$ref: '#/components/schemas/Metric'
ProcessingSummary:
type: object
properties:
timeseries_queue_size:
description: >-
Indicates the total timeseries messages in the queue (to be
processed)
type: integer
format: int64
example: 100
relations_queue_size:
description: >-
Indicates the total relations messages in the queue (to be
processed)
type: integer
format: int64
example: 100
events_queue_size:
description: Indicates the total events messages in the queue (to be processed)
type: integer
format: int64
example: 100
tags_queue_size:
description: Indicates the total tags messages in the queue (to be processed)
type: integer
format: int64
example: 100
oldest_timeseries_insert_date:
description: >
Indicates the oldest timeseries message to be processed. Used to
calculate the age related to the current time. In case o no date the
value
+999999999-12-31T23:59:59.999999999-18:00 will be returned
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
oldest_relations_insert_date:
description: >
Indicates the oldest relations message to be processed. Used to
calculate the age related to the current time. In case o no date the
value
+999999999-12-31T23:59:59.999999999-18:00 will be returned
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
oldest_events_insert_date:
description: >
Indicates the oldest events message to be processed. Used to
calculate the age related to the current time. In case o no date the
value
+999999999-12-31T23:59:59.999999999-18:00 will be returned
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
oldest_tags_insert_date:
description: >
Indicates the oldest tags message to be processed. Used to calculate
the age related to the current time. In case o no date the value
+999999999-12-31T23:59:59.999999999-18:00 will be returned
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
ProcessingDetail:
type: object
properties:
provider_id:
description: Indicated the provider id. The counts are aggregated by provider id.
type: integer
format: int32
example: 5
total_timeseries_messages:
description: Indicates the total timeseries messages loaded by provider
type: integer
format: int64
example: 100
total_relations_messages:
description: Indicates the total relations messages loaded by provider
type: integer
format: int64
example: 100
total_events_messages:
description: Indicates the total events messages loaded by provider
type: integer
format: int64
example: 100
total_tags_messages:
description: Indicates the total tags messages loaded by provider
type: integer
format: int64
example: 100
remain_timeseries_messages:
description: Indicates the remain timeseries messages to process
type: integer
format: int64
example: 100
remain_relations_messages:
description: Indicates the remain relations messages to process
type: integer
format: int64
example: 100
remain_events_messages:
description: Indicates the remain events messages to process
type: integer
format: int64
example: 100
remain_tags_messages:
description: Indicates the remain tags messages to process
type: integer
format: int64
example: 100
start_time:
description: Indicates the time when the first message was loaded by the provider
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
last_update:
description: Indicates the time when the last message has been processed
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
EventData:
type: object
required:
- name
- type_id
- lookup
properties:
name:
description: The name of the event
type: string
example: Host disconnected from cluster
description:
description: The name of the event
type: string
example: >-
Disconnected from pe-pun-bco-dv05.bmc.com in TSCO-Datastore. Reason:
User request
type_id:
description: >
Type of the event: 1=Not planned incident; 2=Planned generic;
3=Planned maintenance; 4=Planned change; 5=Not planned problem;
6=Not planned error; 7=Unknown
type: integer
minimum: 1
maximum: 7
example: 1
note:
description: a note associated to the event
type: string
example: Request from user 12345
creation_time:
description: time when the event has happened
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
source_event_id:
description: event identifier for the source that generate the event
type: string
example: '19'
severity:
description: level of severity
type: string
default: MINOR
example: OK
enum:
- UNKNOWN
- OK
- INFO
- WARNING
- MINOR
- MAJOR
- CRITICAL
source_class:
description: Source class
type: string
example: HOST_STATUS
service_unavailability:
description: Can cause a service unavailability
type: boolean
example: true
occurrences:
description: Count of occurence
type: integer
example: 3
intervent_time:
description: time when the event has been processed
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
resolution_time:
description: time when the event has been resolved
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
pending_time:
description: Times between creation_time and resolution_time
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
state_transition_time:
description: State transition time
type: string
format: date-time
example: 2020-01-01T00:00:00.000Z
entity_name:
description: The entity name associated to the event
type: string
example: R&D ESX host 1
entity_type:
description: >
The entity category and type specification.
String contains the category (sys, bd, dom) encoded with entity type
(like vhc:vmw).
Specification :
type: string
pattern: '^(sys|app|wkld)(:{1}[a-z]+){1,}$'
example: 'sys:vhc:vmw'
metadata:
type: object
required:
- etl_id
- provider_id
- entity_catalog_id
- import_id
properties:
etl_id:
type: integer
example: 16
provider_id:
type: integer
example: 4
entity_catalog_id:
type: integer
example: 37
import_id:
description: >-
The identifier for the data import. It is generated by the
Loader and used to group notifications when data is stored.
type: string
format: uuid
example: eeb8b3d4-e6a7-11ea-adc1-0242ac120002
skip_entities_creation:
type: boolean
description: 'if true, no new entities are created'
default: false
remove_domain_from_lookup:
type: boolean
description: remove domain from lookup
default: false
remove_domain_from_sysnm:
type: boolean
description: remove domain from system name
default: true
lookup:
description: >-
The array of key/value to use as look up. Lookup keys are processed
in specified order and if a certain position is producing an hit
previous keys should not be contradicted.
type: array
items:
$ref: '#/components/schemas/Lookup'
responses:
Unauthorized:
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
Forbidden:
description: Forbidden
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
NotFound:
description: Not Found
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
BadRequest:
description: Bad request
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
InternalServerError:
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
SummaryProcessingResponse:
description: Summary processing information
content:
application/json:
schema:
$ref: '#/components/schemas/ProcessingSummary'
example:
timeseries_queue_size: 52
relations_queue_size: 3
oldest_timeseries_insert_date: 2021-03-25T00:00:00.000Z
oldest_relations_insert_date: 2021-03-25T01:00:00.000Z
DetailProcessingResponse:
description: Detail processing information
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/ProcessingDetail'
example:
- provider_id: 5
total_timeseries_messages: 100
total_relations_messages: 30
remain_timeseries_messages: 50
remain_relations_messages: 20
start_time: 2021-03-25T00:00:00.000Z
last_update: 2021-03-25T01:00:00.000Z
- provider_id: 7
total_timeseries_messages: 200
total_relations_messages: 37
remain_timeseries_messages: 40
remain_relations_messages: 29
start_time: 2021-03-24T00:00:00.000Z
last_update: 2021-03-25T01:00:00.000Z
tags:
- name: relation
description: Ingest for relations
- name: about
description: The status of the service
- name: tags
description: Ingest tags information
- name: ingestion
description: Ingest for timeseries data
- name: observability
description: Ingest processing monitoring
paths:
/opt/api/htsingest/about:
get:
tags:
- about
summary: getAboutInfo
operationId: about
security: []
responses:
'200':
description: Ok
content:
application/json:
schema:
$ref: '#/components/schemas/About'
'401':
$ref: '#/components/responses/Unauthorized'
'403':
$ref: '#/components/responses/Forbidden'
'404':
$ref: '#/components/responses/NotFound'
deprecated: false
/opt/api/v1/htsingest/relations:
post:
summary: Upload object relation (objrel) gz flat file archive for processing
description: >
Upload object relation (objrel) gz flat file archive for processing.
Receive the stream of gzip file.
Objrel flat file archive is a gzip file composed by:
* metadata.properties
* objrel.csv
### Metadata file
It's a property files that contains:
* change.mode: the type of objrel (ASSERT|CHANGE)
* load.ts: date of loading
* subsetid: if present, groups entities and relationships that must be
compared together. Used by some ETL
* namespace: the namespace
* data.header: the header of the objrel.csv file
* source.id: id of the source ETL
* task.id: task id that imported the data
Example:
```properties
change.mode=ASSERT
load.ts=2020-05-11T17\:13\:42.328+02\:00
subsetid=3
namespace=opt
data.header=ENTCATNM;DS_ENTNM_HASH;ENTCATNMPARENT;DS_ENTNMPARENT_HASH;RELATIONTYPE;RELATIONMODE;ENTNM;ENTTYPENM;DESCRIPTION;LOCATIONNM;ENTLOOKUP;ENTLOOKUPPARENT;STRONGLOOKUPFIELDS;WEAKLOOKUPFIELDS;TS;CHANGETYPE
source.id=1
task.id=1
```
### Objrel file
It's a csv files that contains objrel dataset data
Example:
```csv
SYS;;0025ab61c40b065982b677b24a6f2bc2;;;0;Isreal-host;vhc:vmw;;;NAME#Isreal-host##PARENT_VCNAME#BCM-VCENTER##PARENT_VCUUID#63BA5246-A098-407F-B797-89E5E1B145D4##VMW_VMREF#domain-c2902##_COMPATIBILITY_#Isreal-host;;PARENT_VCUUID&&VMW_VMREF;PARENT_VCNAME&&PARENT_FOLDER&&NAME##PARENT_VCNAME&&NAME##NAME;2020-05-08
06:00:00;ASSERT
```
tags:
- relation
operationId: relation
parameters:
- in: query
name: mode
required: true
schema:
$ref: '#/components/schemas/RelationMode'
- in: query
name: import_id
required: false
schema:
type: string
format: uuid
- in: query
name: provider_id
required: false
schema:
type: number
- in: query
name: total_messages
required: false
schema:
type: number
requestBody:
content:
application/octet-stream:
schema:
type: string
format: binary
responses:
'201':
description: OK
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'
/opt/api/v1/htsingest/tags:
post:
summary: Upload object tags gz flat file archive for processing
description: >
Upload object tags gz flat file archive for processing. Receive the
stream of gzip file.
Tags flat file archive is a gzip file composed by:
* metadata.properties
* tags.csv
### Metadata file
It's a property files that contains:
* load.ts: date of loading
* data.header: the header of the objrel.csv file
* source.id: id of the source ETL
* task.id: task id that imported the data
Example:
```properties
data.header=ENTCATNM;ENTNM;DS_ENTNM;TAGTYPE;TAGTYPEDESCR;ENTTYPENM;TAG;TAGDESCR;STRONGLOOKUPFIELDS;WEAKLOOKUPFIELDS;OPERATION
load.ts=2020-08-14T15\:58\:20.932445+05\:30
task.id=155
source.id=103
```
### Tags file
It's a csv files that contains objrel dataset data
Example:
```csv
SYS;server1;HOSNAME#server1.bmc.com;Environment;;sys:gm:vmw;Production;HOSTNAME;;ADD
```
tags:
- tags
operationId: tags
parameters:
- in: query
name: import_id
required: false
schema:
type: string
format: uuid
- in: query
name: provider_id
required: false
schema:
type: number
- in: query
name: total_messages
required: false
schema:
type: number
requestBody:
content:
application/octet-stream:
schema:
type: string
format: binary
responses:
'201':
description: OK
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'
/opt/api/v1/htsingest/timeseries:
post:
summary: Upload time series for the specified entity
tags:
- ingestion
operationId: uploadTimeseries
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/TimeseriesData'
responses:
'201':
description: OK
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'
/opt/api/v1/htsingest/timeseries/bulk:
post:
summary: Upload time series for the specified entities
tags:
- ingestion
operationId: bulkUploadTimeseries
requestBody:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/TimeseriesData'
responses:
'201':
description: OK
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'
/opt/api/v1/htsingest/observability/summary:
get:
summary: Return summary information about timeseries and relation processing
tags:
- observability
operationId: summaryProcessing
responses:
'200':
$ref: '#/components/responses/SummaryProcessingResponse'
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'
/opt/api/v1/htsingest/observability/details:
get:
summary: >-
Return detail information about timeseries and relation processing by
ETls
tags:
- observability
operationId: detailProcessing
responses:
'200':
$ref: '#/components/responses/DetailProcessingResponse'
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'
/opt/api/v1/htsingest/events/bulk:
post:
summary: Upload events data
tags:
- ingestion
operationId: bulkUploadEvent
requestBody:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/EventData'
responses:
'201':
description: OK
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/InternalServerError'