Class WorkerConfiguration

(experimental) The worker configuration for a Spark job.

Inheritance

object

WorkerConfiguration

Implements

IWorkerConfiguration

Inherited Members

object.GetType()

object.MemberwiseClone()

object.ToString()

object.Equals(object)

object.Equals(object, object)

object.ReferenceEquals(object, object)

object.GetHashCode()

Namespace: Amazon.CDK.AWS.Glue.Alpha

Assembly: Amazon.CDK.AWS.Glue.Alpha.dll

Syntax (csharp)

public class WorkerConfiguration : IWorkerConfiguration

Syntax (vb)

Public Class WorkerConfiguration Implements IWorkerConfiguration

Remarks

The worker type and the number of workers are set together: providing this configuration requires both values, so a Spark job can never be given one without the other.

Stability: Experimental

ExampleMetadata: infused

Examples

using Amazon.CDK;
             using Amazon.CDK.AWS.IAM;
             Stack stack;
             IRole role;
             Code script;

             new PySparkEtlJob(stack, "PySparkETLJob", new PySparkEtlJobProps {
                 JobName = "PySparkETLJobCustomName",
                 Description = "This is a description",
                 Role = role,
                 Script = script,
                 GlueVersion = GlueVersion.V5_1,
                 ContinuousLogging = new ContinuousLoggingProps { Enabled = false },
                 WorkerConfiguration = new WorkerConfiguration {
                     WorkerType = WorkerType.G_2X,
                     NumberOfWorkers = 2
                 },
                 MaxConcurrentRuns = 100,
                 Timeout = Duration.Hours(2),
                 Connections = new [] { Connection.FromConnectionName(stack, "Connection", "connectionName") },
                 SecurityConfiguration = SecurityConfiguration.FromSecurityConfigurationName(stack, "SecurityConfig", "securityConfigName"),
                 Tags = new Dictionary<string, string> {
                     { "FirstTagName", "FirstTagValue" },
                     { "SecondTagName", "SecondTagValue" },
                     { "XTagName", "XTagValue" }
                 },
                 MaxRetries = 2
             });

Synopsis

Constructors

WorkerConfiguration()

(experimental) The worker configuration for a Spark job.

Properties

NumberOfWorkers	(experimental) The number of workers of the given workerType that are allocated when a job runs.
WorkerType	(experimental) The type of predefined worker that is allocated when a job runs.

Constructors

WorkerConfiguration()

(experimental) The worker configuration for a Spark job.

public WorkerConfiguration()

Remarks

The worker type and the number of workers are set together: providing this configuration requires both values, so a Spark job can never be given one without the other.

Stability: Experimental

ExampleMetadata: infused

Examples

using Amazon.CDK;
             using Amazon.CDK.AWS.IAM;
             Stack stack;
             IRole role;
             Code script;

             new PySparkEtlJob(stack, "PySparkETLJob", new PySparkEtlJobProps {
                 JobName = "PySparkETLJobCustomName",
                 Description = "This is a description",
                 Role = role,
                 Script = script,
                 GlueVersion = GlueVersion.V5_1,
                 ContinuousLogging = new ContinuousLoggingProps { Enabled = false },
                 WorkerConfiguration = new WorkerConfiguration {
                     WorkerType = WorkerType.G_2X,
                     NumberOfWorkers = 2
                 },
                 MaxConcurrentRuns = 100,
                 Timeout = Duration.Hours(2),
                 Connections = new [] { Connection.FromConnectionName(stack, "Connection", "connectionName") },
                 SecurityConfiguration = SecurityConfiguration.FromSecurityConfigurationName(stack, "SecurityConfig", "securityConfigName"),
                 Tags = new Dictionary<string, string> {
                     { "FirstTagName", "FirstTagValue" },
                     { "SecondTagName", "SecondTagValue" },
                     { "XTagName", "XTagValue" }
                 },
                 MaxRetries = 2
             });

Properties

NumberOfWorkers

(experimental) The number of workers of the given workerType that are allocated when a job runs.

public double NumberOfWorkers { get; set; }

Property Value

double

Remarks

Stability: Experimental

WorkerType

(experimental) The type of predefined worker that is allocated when a job runs.

public WorkerType WorkerType { get; set; }

Property Value

WorkerType

Remarks

Enum options: Standard, G_1X, G_2X, G_025X, G_4X, G_8X, Z_2X

Stability: Experimental

Implements

IWorkerConfiguration