Interface ISparkJobProps

Common properties for different types of Spark jobs.

Inherited Members
Namespace: Amazon.CDK.AWS.Glue
Assembly: Amazon.CDK.Lib.dll
Syntax (csharp)
public interface ISparkJobProps : IJobProps
Syntax (vb)
Public Interface ISparkJobProps Inherits IJobProps
Remarks

ExampleMetadata: fixture=_generated

Examples
// The code below shows an example of how to instantiate this type.
            // The values are placeholders you should change.
            using Amazon.CDK;
            using Amazon.CDK.AWS.Glue;
            using Amazon.CDK.AWS.IAM;
            using Amazon.CDK.AWS.Logs;
            using Amazon.CDK.AWS.S3;

            Bucket bucket;
            Code code;
            Connection connection;
            LogGroup logGroup;
            Role role;
            SecurityConfiguration securityConfiguration;

            var sparkJobProps = new SparkJobProps {
                Role = role,
                Script = code,

                // the properties below are optional
                Connections = new [] { connection },
                ContinuousLogging = new ContinuousLoggingProps {
                    Enabled = false,

                    // the properties below are optional
                    ConversionPattern = "conversionPattern",
                    LogGroup = logGroup,
                    LogStreamPrefix = "logStreamPrefix",
                    Quiet = false
                },
                DefaultArguments = new Dictionary<string, string> {
                    { "defaultArgumentsKey", "defaultArguments" }
                },
                Description = "description",
                EnableMetrics = false,
                EnableObservabilityMetrics = false,
                GlueVersion = GlueVersion.V0_9,
                JobName = "jobName",
                MaxConcurrentRuns = 123,
                MaxRetries = 123,
                SecurityConfiguration = securityConfiguration,
                SparkUI = new SparkUIProps {
                    Bucket = bucket,
                    Prefix = "prefix"
                },
                Tags = new Dictionary<string, string> {
                    { "tagsKey", "tags" }
                },
                Timeout = Duration.Minutes(30),
                WorkerConfiguration = new WorkerConfiguration {
                    NumberOfWorkers = 123,
                    WorkerType = WorkerType.STANDARD
                }
            };

Synopsis

Properties

EnableMetrics

Enable profiling metrics for the Glue job.
EnableObservabilityMetrics

Enable observability metrics for the Glue job.
SparkUI

Enables the Spark UI debugging and monitoring with the specified props.
WorkerConfiguration

The worker type and the number of workers allocated when a job runs.

Properties

EnableMetrics

Enable profiling metrics for the Glue job.

bool? EnableMetrics { get; }
Property Value

bool?

Remarks

When enabled, adds '--enable-metrics' to job arguments.

Default: true

EnableObservabilityMetrics

Enable observability metrics for the Glue job.

bool? EnableObservabilityMetrics { get; }
Property Value

bool?

Remarks

When enabled, adds '--enable-observability-metrics': 'true' to job arguments.

Default: true

SparkUI

Enables the Spark UI debugging and monitoring with the specified props.

ISparkUIProps? SparkUI { get; }
Property Value

ISparkUIProps

Remarks

Default: - Spark UI debugging and monitoring is disabled.

See: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html

WorkerConfiguration

The worker type and the number of workers allocated when a job runs.

IWorkerConfiguration? WorkerConfiguration { get; }
Property Value

IWorkerConfiguration

Remarks

Default: - the job runs with the G_1X worker type and 10 workers.