⚝
One Hat Cyber Team
⚝
Your IP:
216.73.216.118
Server IP:
77.37.115.141
Server:
Linux sg-nme-web1518.main-hosting.eu 5.14.0-611.16.1.el9_7.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Dec 22 03:40:39 EST 2025 x86_64
Server Software:
LiteSpeed
PHP Version:
8.3.28
Buat File
|
Buat Folder
Eksekusi
Dir :
~
/
opt
/
gsutil
/
gslib
/
vendored
/
boto
/
tests
/
unit
/
emr
/
View File Name :
test_emr_responses.py
# Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com> # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # NOTE: These tests only cover the very simple cases I needed to test # for the InstanceGroup fix. import xml.sax from boto import handler from boto.emr import emrobject from boto.resultset import ResultSet from tests.compat import unittest JOB_FLOW_EXAMPLE = b""" <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15"> <DescribeJobFlowsResult> <JobFlows> <member> <ExecutionStatusDetail> <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime> <StartDateTime>2009-01-28T21:49:16Z</StartDateTime> <State>STARTING</State> </ExecutionStatusDetail> <BootstrapActions> <member> <BootstrapActionConfig> <ScriptBootstrapAction> <Args/> <Path>s3://elasticmapreduce/libs/hue/install-hue</Path> </ScriptBootstrapAction> <Name>Install Hue</Name> </BootstrapActionConfig> </member> </BootstrapActions> <VisibleToAllUsers>true</VisibleToAllUsers> <SupportedProducts> <member>Hue</member> </SupportedProducts> <Name>MyJobFlowName</Name> <LogUri>mybucket/subdir/</LogUri> <Steps> <member> <ExecutionStatusDetail> <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime> <State>PENDING</State> </ExecutionStatusDetail> <StepConfig> <HadoopJarStep> <Jar>MyJarFile</Jar> <MainClass>MyMailClass</MainClass> <Args> <member>arg1</member> <member>arg2</member> </Args> <Properties/> </HadoopJarStep> <Name>MyStepName</Name> <ActionOnFailure>CONTINUE</ActionOnFailure> </StepConfig> </member> </Steps> <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId> <Instances> <Placement> <AvailabilityZone>us-east-1a</AvailabilityZone> </Placement> <SlaveInstanceType>m1.small</SlaveInstanceType> <MasterInstanceType>m1.small</MasterInstanceType> <Ec2KeyName>myec2keyname</Ec2KeyName> <InstanceCount>4</InstanceCount> <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps> </Instances> </member> </JobFlows> </DescribeJobFlowsResult> <ResponseMetadata> <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId> </ResponseMetadata> </DescribeJobFlowsResponse> """ JOB_FLOW_COMPLETED = b""" <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <DescribeJobFlowsResult> <JobFlows> <member> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <LastStateChangeReason>Steps completed</LastStateChangeReason> <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> </ExecutionStatusDetail> <BootstrapActions/> <Name>RealJobFlowName</Name> <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri> <Steps> <member> <StepConfig> <HadoopJarStep> <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar> <Args> <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch</member> </Args> <Properties/> </HadoopJarStep> <Name>Setup Hadoop Debugging</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:04:22Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/20/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/19/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/18/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/17/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/16/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/15/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/14/*</member> <member>-output</member> <member>s3://example.emrtest.crunched/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_Initial</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:04:22Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:36:18Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.crunched/*</member> <member>-output</member> <member>s3://example.emrtest.step1/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step1</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:36:18Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:37:51Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.crunched/*</member> <member>-output</member> <member>s3://example.emrtest.step2/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step2</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:37:51Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:39:32Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.step1/*</member> <member>-output</member> <member>s3://example.emrtest.step3/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step3</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:39:32Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:41:22Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.step1/*</member> <member>-output</member> <member>s3://example.emrtest.step4/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step4</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:41:22Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:43:03Z</EndDateTime> </ExecutionStatusDetail> </member> </Steps> <JobFlowId>j-3H3Q13JPFLU22</JobFlowId> <Instances> <SlaveInstanceType>m1.large</SlaveInstanceType> <MasterInstanceId>i-64c21609</MasterInstanceId> <Placement> <AvailabilityZone>us-east-1b</AvailabilityZone> </Placement> <InstanceGroups> <member> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <InstanceRunningCount>0</InstanceRunningCount> <StartDateTime>2010-10-21T01:02:09Z</StartDateTime> <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime> <State>ENDED</State> <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> <InstanceRequestCount>1</InstanceRequestCount> <InstanceType>m1.large</InstanceType> <Market>ON_DEMAND</Market> <LastStateChangeReason>Job flow terminated</LastStateChangeReason> <InstanceRole>MASTER</InstanceRole> <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId> <Name>master</Name> </member> <member> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <InstanceRunningCount>0</InstanceRunningCount> <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime> <State>ENDED</State> <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> <InstanceRequestCount>9</InstanceRequestCount> <InstanceType>m1.large</InstanceType> <Market>ON_DEMAND</Market> <LastStateChangeReason>Job flow terminated</LastStateChangeReason> <InstanceRole>CORE</InstanceRole> <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId> <Name>slave</Name> </member> </InstanceGroups> <NormalizedInstanceHours>40</NormalizedInstanceHours> <HadoopVersion>0.20</HadoopVersion> <MasterInstanceType>m1.large</MasterInstanceType> <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</MasterPublicDnsName> <Ec2KeyName>myubersecurekey</Ec2KeyName> <InstanceCount>10</InstanceCount> <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps> </Instances> </member> </JobFlows> </DescribeJobFlowsResult> <ResponseMetadata> <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId> </ResponseMetadata> </DescribeJobFlowsResponse> """ class TestEMRResponses(unittest.TestCase): def _parse_xml(self, body, markers): rs = ResultSet(markers) h = handler.XmlHandler(rs, None) xml.sax.parseString(body, h) return rs def _assert_fields(self, response, **fields): for field, expected in fields.items(): actual = getattr(response, field) self.assertEqual(expected, actual, "Field %s: %r != %r" % (field, expected, actual)) def test_JobFlows_example(self): [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE, [('member', emrobject.JobFlow)]) self._assert_fields(jobflow, creationdatetime='2009-01-28T21:49:16Z', startdatetime='2009-01-28T21:49:16Z', state='STARTING', instancecount='4', jobflowid='j-3UN6WX5RRO2AG', loguri='mybucket/subdir/', name='MyJobFlowName', availabilityzone='us-east-1a', slaveinstancetype='m1.small', masterinstancetype='m1.small', ec2keyname='myec2keyname', keepjobflowalivewhennosteps='true') def test_JobFlows_completed(self): [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED, [('member', emrobject.JobFlow)]) self._assert_fields(jobflow, creationdatetime='2010-10-21T01:00:25Z', startdatetime='2010-10-21T01:03:59Z', enddatetime='2010-10-21T01:44:18Z', state='COMPLETED', instancecount='10', jobflowid='j-3H3Q13JPFLU22', loguri='s3n://example.emrtest.scripts/jobflow_logs/', name='RealJobFlowName', availabilityzone='us-east-1b', slaveinstancetype='m1.large', masterinstancetype='m1.large', ec2keyname='myubersecurekey', keepjobflowalivewhennosteps='false') self.assertEqual(6, len(jobflow.steps)) self.assertEqual(2, len(jobflow.instancegroups))