Pipeline Service¶

{
  "$id": "https://open-metadata.org/schema/entity/services/pipelineService.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "Pipeline Service",
  "description": "This schema defines the Pipeline Service entity, such as Airflow and Prefect.",
  "type": "object",
  "javaType": "org.openmetadata.schema.entity.services.PipelineService",
  "javaInterfaces": [
    "org.openmetadata.schema.EntityInterface",
    "org.openmetadata.schema.ServiceEntityInterface"
  ],

  "definitions": {
    "pipelineServiceType": {
      "description": "Type of pipeline service - Airflow or Prefect.",
      "type": "string",
      "javaInterfaces": [
        "org.openmetadata.schema.EnumInterface"
      ],
      "enum": [
        "Airflow", "GluePipeline", "KinesisFirehose",
        "Airbyte", "Fivetran", "Flink", "Dagster",
        "Nifi", "DomoPipeline", "CustomPipeline",
        "DatabricksPipeline", "Spline", "Spark",
        "OpenLineage", "KafkaConnect", "DBTCloud",
        "Matillion", "Stitch", "DataFactory",
        "Wherescape", "SSIS", "Snowplow"
      ]
    },
    "pipelineConnection": {
      "type": "object",
      "javaType": "org.openmetadata.schema.type.PipelineConnection",
      "description": "Pipeline Connection.",
      "javaInterfaces": [
        "org.openmetadata.schema.ServiceConnectionEntityInterface"
      ],
      "properties": {
        "config": {
          "mask": true,
          "oneOf": [
            {"$ref": "./connections/pipeline/airflowConnection.json"},
            {"$ref": "./connections/pipeline/wherescapeConnection.json"},
            {"$ref": "./connections/pipeline/ssisConnection.json"},
            {"$ref": "./connections/pipeline/gluePipelineConnection.json"},
            {"$ref": "./connections/pipeline/kinesisFirehoseConnection.json"},
            {"$ref": "./connections/pipeline/airbyteConnection.json"},
            {"$ref": "./connections/pipeline/fivetranConnection.json"},
            {"$ref": "./connections/pipeline/flinkConnection.json"},
            {"$ref": "./connections/pipeline/dagsterConnection.json"},
            {"$ref": "./connections/pipeline/nifiConnection.json"},
            {"$ref": "./connections/pipeline/domoPipelineConnection.json"},
            {"$ref": "./connections/pipeline/customPipelineConnection.json"},
            {"$ref": "./connections/pipeline/databricksPipelineConnection.json"},
            {"$ref": "./connections/pipeline/splineConnection.json"},
            {"$ref": "./connections/pipeline/sparkConnection.json"},
            {"$ref": "./connections/pipeline/openLineageConnection.json"},
            {"$ref": "./connections/pipeline/kafkaConnectConnection.json"},
            {"$ref": "./connections/pipeline/dbtCloudConnection.json"},
            {"$ref": "./connections/pipeline/matillionConnection.json"},
            {"$ref": "./connections/pipeline/datafactoryConnection.json"},
            {"$ref": "./connections/pipeline/stitchConnection.json"},
            {"$ref": "./connections/pipeline/snowplowConnection.json"}
          ]
        }
      },
      "additionalProperties": false
    }
  },

  "properties": {
    "id": {
      "description": "Unique identifier of this pipeline service instance.",
      "$ref": "../../type/basic.json#/definitions/uuid"
    },
    "name": {
      "description": "Name that identifies this pipeline service.",
      "$ref": "../../type/basic.json#/definitions/entityName"
    },
    "fullyQualifiedName": {
      "description": "FullyQualifiedName same as `name`.",
      "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
    },
    "serviceType": {
      "description": "Type of pipeline service such as Airflow or Prefect...",
      "$ref": "#/definitions/pipelineServiceType"
    },
    "description": {
      "description": "Description of a pipeline service instance.",
      "type": "string"
    },
    "displayName": {
      "description": "Display Name that identifies this pipeline service.",
      "type": "string"
    },
    "version": {
      "description": "Metadata version of the entity.",
      "$ref": "../../type/entityHistory.json#/definitions/entityVersion"
    },
    "updatedAt": {
      "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
      "$ref": "../../type/basic.json#/definitions/timestamp"
    },
    "updatedBy": {
      "description": "User who made the update.",
      "type": "string"
    },
    "impersonatedBy": {
      "description": "Bot user that performed the action on behalf of the actual user.",
      "$ref": "../../type/basic.json#/definitions/impersonatedBy"
    },
    "testConnectionResult": {
      "description": "Last test connection results for this service",
      "$ref": "connections/testConnectionResult.json"
    },
    "tags": {
      "description": "Tags for this Pipeline Service.",
      "type": "array",
      "items": {
        "$ref": "../../type/tagLabel.json"
      },
      "default": []
    },
    "pipelines": {
      "description": "References to pipelines deployed for this pipeline service to extract metadata",
      "$ref": "../../type/entityReferenceList.json"
    },
    "connection": {
      "$ref": "#/definitions/pipelineConnection"
    },
    "owners": {
      "description": "Owners of this pipeline service.",
      "$ref": "../../type/entityReferenceList.json"
    },
    "href": {
      "description": "Link to the resource corresponding to this pipeline service.",
      "$ref": "../../type/basic.json#/definitions/href"
    },
    "changeDescription": {
      "description": "Change that lead to this version of the entity.",
      "$ref": "../../type/entityHistory.json#/definitions/changeDescription"
    },
    "incrementalChangeDescription": {
      "description": "Change that lead to this version of the entity.",
      "$ref": "../../type/entityHistory.json#/definitions/changeDescription"
    },
    "deleted": {
      "description": "When `true` indicates the entity has been soft deleted.",
      "type": "boolean",
      "default": false
    },
    "dataProducts": {
      "description": "List of data products this entity is part of.",
      "$ref": "../../type/entityReferenceList.json"
    },
    "followers": {
      "description": "Followers of this entity.",
      "$ref": "../../type/entityReferenceList.json"
    },
    "domains": {
      "description": "Domains the Pipeline service belongs to.",
      "$ref": "../../type/entityReferenceList.json"
    },
    "ingestionRunner": {
      "description": "The ingestion agent responsible for executing the ingestion pipeline.",
      "$ref": "../../type/entityReference.json"
    }
  },

  "required": ["id", "name", "serviceType"],
  "additionalProperties": false
}

{
  "@context": {
    "@vocab": "https://open-metadata.org/schema/",
    "om": "https://open-metadata.org/schema/",
    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
    "xsd": "http://www.w3.org/2001/XMLSchema#",

    "PipelineService": "om:PipelineService",
    "name": {
      "@id": "om:pipelineServiceName",
      "@type": "xsd:string"
    },
    "fullyQualifiedName": {
      "@id": "om:fullyQualifiedName",
      "@type": "xsd:string"
    },
    "displayName": {
      "@id": "om:displayName",
      "@type": "xsd:string"
    },
    "description": {
      "@id": "om:description",
      "@type": "xsd:string"
    },
    "serviceType": {
      "@id": "om:serviceType",
      "@type": "@vocab"
    },
    "connection": {
      "@id": "om:hasConnection",
      "@type": "@id"
    },
    "pipelines": {
      "@id": "om:hasPipeline",
      "@type": "@id",
      "@container": "@set"
    },
    "owners": {
      "@id": "om:hasOwner",
      "@type": "@id",
      "@container": "@set"
    },
    "domains": {
      "@id": "om:belongsToDomain",
      "@type": "@id",
      "@container": "@set"
    },
    "tags": {
      "@id": "om:hasTag",
      "@type": "@id",
      "@container": "@set"
    },
    "dataProducts": {
      "@id": "om:partOfDataProduct",
      "@type": "@id",
      "@container": "@set"
    },
    "followers": {
      "@id": "om:hasFollower",
      "@type": "@id",
      "@container": "@set"
    },
    "deleted": {
      "@id": "om:deleted",
      "@type": "xsd:boolean"
    },
    "href": {
      "@id": "om:href",
      "@type": "xsd:anyURI"
    },
    "ingestionRunner": {
      "@id": "om:hasIngestionRunner",
      "@type": "@id"
    }
  }
}

{
  "@context": "https://open-metadata.org/context/pipelineService.jsonld",
  "@type": "PipelineService",
  "@id": "https://example.com/services/airflow_prod",

  "name": "airflow_prod",
  "fullyQualifiedName": "airflow_prod",
  "displayName": "Production Airflow",
  "description": "Production Airflow instance for ETL workflows",
  "serviceType": "Airflow",

  "connection": {
    "config": {
      "type": "Airflow",
      "hostPort": "https://airflow.company.com",
      "numberOfStatus": 10,
      "connection": {
        "type": "Backend"
      }
    }
  },

  "owners": [
    {
      "@id": "https://example.com/teams/data-engineering",
      "@type": "Team",
      "name": "data-engineering",
      "displayName": "Data Engineering"
    }
  ],

  "domains": [
    {
      "@id": "https://example.com/domains/engineering",
      "@type": "Domain",
      "name": "Engineering"
    }
  ],

  "tags": [
    {
      "@id": "https://open-metadata.org/tags/Tier/Gold",
      "tagFQN": "Tier.Gold"
    },
    {
      "@id": "https://open-metadata.org/tags/Environment/Production",
      "tagFQN": "Environment.Production"
    }
  ],

  "pipelines": [
    {
      "@id": "https://example.com/pipelines/customer_etl",
      "@type": "Pipeline",
      "name": "customer_etl"
    },
    {
      "@id": "https://example.com/pipelines/sales_reporting",
      "@type": "Pipeline",
      "name": "sales_reporting"
    }
  ],

  "followers": [
    {
      "@id": "https://example.com/users/john.doe",
      "@type": "User",
      "name": "john.doe"
    }
  ],

  "deleted": false
}