티스토리 뷰

공부

[emr] RunJobFlow json

승가비 2023. 6. 17. 09:28
728x90
{
   "AdditionalInfo": "string",
   "AmiVersion": "string",
   "Applications": [ 
      { 
         "AdditionalInfo": { 
            "string" : "string" 
         },
         "Args": [ "string" ],
         "Name": "string",
         "Version": "string"
      }
   ],
   "AutoScalingRole": "string",
   "AutoTerminationPolicy": { 
      "IdleTimeout": number
   },
   "BootstrapActions": [ 
      { 
         "Name": "string",
         "ScriptBootstrapAction": { 
            "Args": [ "string" ],
            "Path": "string"
         }
      }
   ],
   "Configurations": [ 
      { 
         "Classification": "string",
         "Configurations": [ 
            "Configuration"
         ],
         "Properties": { 
            "string" : "string" 
         }
      }
   ],
   "CustomAmiId": "string",
   "EbsRootVolumeSize": number,
   "Instances": { 
      "AdditionalMasterSecurityGroups": [ "string" ],
      "AdditionalSlaveSecurityGroups": [ "string" ],
      "Ec2KeyName": "string",
      "Ec2SubnetId": "string",
      "Ec2SubnetIds": [ "string" ],
      "EmrManagedMasterSecurityGroup": "string",
      "EmrManagedSlaveSecurityGroup": "string",
      "HadoopVersion": "string",
      "InstanceCount": number,
      "InstanceFleets": [ 
         { 
            "InstanceFleetType": "string",
            "InstanceTypeConfigs": [ 
               { 
                  "BidPrice": "string",
                  "BidPriceAsPercentageOfOnDemandPrice": number,
                  "Configurations": [ 
                     { 
                        "Classification": "string",
                        "Configurations": [ 
                           "Configuration"
                        ],
                        "Properties": { 
                           "string" : "string" 
                        }
                     }
                  ],
                  "CustomAmiId": "string",
                  "EbsConfiguration": { 
                     "EbsBlockDeviceConfigs": [ 
                        { 
                           "VolumeSpecification": { 
                              "Iops": number,
                              "SizeInGB": number,
                              "Throughput": number,
                              "VolumeType": "string"
                           },
                           "VolumesPerInstance": number
                        }
                     ],
                     "EbsOptimized": boolean
                  },
                  "InstanceType": "string",
                  "WeightedCapacity": number
               }
            ],
            "LaunchSpecifications": { 
               "OnDemandSpecification": { 
                  "AllocationStrategy": "string",
                  "CapacityReservationOptions": { 
                     "CapacityReservationPreference": "string",
                     "CapacityReservationResourceGroupArn": "string",
                     "UsageStrategy": "string"
                  }
               },
               "SpotSpecification": { 
                  "AllocationStrategy": "string",
                  "BlockDurationMinutes": number,
                  "TimeoutAction": "string",
                  "TimeoutDurationMinutes": number
               }
            },
            "Name": "string",
            "ResizeSpecifications": { 
               "OnDemandResizeSpecification": { 
                  "TimeoutDurationMinutes": number
               },
               "SpotResizeSpecification": { 
                  "TimeoutDurationMinutes": number
               }
            },
            "TargetOnDemandCapacity": number,
            "TargetSpotCapacity": number
         }
      ],
      "InstanceGroups": [ 
         { 
            "AutoScalingPolicy": { 
               "Constraints": { 
                  "MaxCapacity": number,
                  "MinCapacity": number
               },
               "Rules": [ 
                  { 
                     "Action": { 
                        "Market": "string",
                        "SimpleScalingPolicyConfiguration": { 
                           "AdjustmentType": "string",
                           "CoolDown": number,
                           "ScalingAdjustment": number
                        }
                     },
                     "Description": "string",
                     "Name": "string",
                     "Trigger": { 
                        "CloudWatchAlarmDefinition": { 
                           "ComparisonOperator": "string",
                           "Dimensions": [ 
                              { 
                                 "Key": "string",
                                 "Value": "string"
                              }
                           ],
                           "EvaluationPeriods": number,
                           "MetricName": "string",
                           "Namespace": "string",
                           "Period": number,
                           "Statistic": "string",
                           "Threshold": number,
                           "Unit": "string"
                        }
                     }
                  }
               ]
            },
            "BidPrice": "string",
            "Configurations": [ 
               { 
                  "Classification": "string",
                  "Configurations": [ 
                     "Configuration"
                  ],
                  "Properties": { 
                     "string" : "string" 
                  }
               }
            ],
            "CustomAmiId": "string",
            "EbsConfiguration": { 
               "EbsBlockDeviceConfigs": [ 
                  { 
                     "VolumeSpecification": { 
                        "Iops": number,
                        "SizeInGB": number,
                        "Throughput": number,
                        "VolumeType": "string"
                     },
                     "VolumesPerInstance": number
                  }
               ],
               "EbsOptimized": boolean
            },
            "InstanceCount": number,
            "InstanceRole": "string",
            "InstanceType": "string",
            "Market": "string",
            "Name": "string"
         }
      ],
      "KeepJobFlowAliveWhenNoSteps": boolean,
      "MasterInstanceType": "string",
      "Placement": { 
         "AvailabilityZone": "string",
         "AvailabilityZones": [ "string" ]
      },
      "ServiceAccessSecurityGroup": "string",
      "SlaveInstanceType": "string",
      "TerminationProtected": boolean
   },
   "JobFlowRole": "string",
   "KerberosAttributes": { 
      "ADDomainJoinPassword": "string",
      "ADDomainJoinUser": "string",
      "CrossRealmTrustPrincipalPassword": "string",
      "KdcAdminPassword": "string",
      "Realm": "string"
   },
   "LogEncryptionKmsKeyId": "string",
   "LogUri": "string",
   "ManagedScalingPolicy": { 
      "ComputeLimits": { 
         "MaximumCapacityUnits": number,
         "MaximumCoreCapacityUnits": number,
         "MaximumOnDemandCapacityUnits": number,
         "MinimumCapacityUnits": number,
         "UnitType": "string"
      }
   },
   "Name": "string",
   "NewSupportedProducts": [ 
      { 
         "Args": [ "string" ],
         "Name": "string"
      }
   ],
   "OSReleaseLabel": "string",
   "PlacementGroupConfigs": [ 
      { 
         "InstanceRole": "string",
         "PlacementStrategy": "string"
      }
   ],
   "ReleaseLabel": "string",
   "RepoUpgradeOnBoot": "string",
   "ScaleDownBehavior": "string",
   "SecurityConfiguration": "string",
   "ServiceRole": "string",
   "StepConcurrencyLevel": number,
   "Steps": [ 
      { 
         "ActionOnFailure": "string",
         "HadoopJarStep": { 
            "Args": [ "string" ],
            "Jar": "string",
            "MainClass": "string",
            "Properties": [ 
               { 
                  "Key": "string",
                  "Value": "string"
               }
            ]
         },
         "Name": "string"
      }
   ],
   "SupportedProducts": [ "string" ],
   "Tags": [ 
      { 
         "Key": "string",
         "Value": "string"
      }
   ],
   "VisibleToAllUsers": boolean
}

https://docs.aws.amazon.com/emr/latest/APIReference/API_RunJobFlow.html

 

RunJobFlow - Amazon EMR

The instance fleets configuration is available only in Amazon EMR releases 4.8.0 and later, excluding 5.0.x versions. The RunJobFlow request can contain InstanceFleets parameters or InstanceGroups parameters, but not both.

docs.aws.amazon.com

https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/emr/client/run_job_flow.html

 

run_job_flow - Boto3 1.28.3 documentation

Previous remove_tags

boto3.amazonaws.com

https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hadoop-task-config.html#w221aac26c21b7c23d115b9

 

Task configuration - Amazon EMR

Task configuration You can set configuration variables to tune the performance of your MapReduce jobs. This section provides the default values for important settings. Default values vary based on the EC2 instance type of the node used in the cluster. HBas

docs.aws.amazon.com

https://github.com/hillday/aws_emr_presto_custom_autoscalling/blob/1c241dc21af947f37e9bb2e636a7afc6545c9ae7/emr/instancegroupconfig.json#L7

 

728x90
저작자표시 비영리

'공부' 카테고리의 다른 글

[sh] exit code  (0) 2023.06.20
[Airflow] render_template_as_native_obj=True (xcom.pull native obj; only str)  (0) 2023.06.17
[sh] sed -i -E "s/old_regexp/new/g" file  (0) 2023.06.17
[sh] change variable to value  (0) 2023.06.17
[datadog] install & yarn  (0) 2023.06.17
댓글