D6927: ci: report cost to run each job
indygreg (Gregory Szorc)
phabricator at mercurial-scm.org
Tue Oct 1 04:41:18 UTC 2019
indygreg updated this revision to Diff 16724.
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D6927?vs=16721&id=16724
CHANGES SINCE LAST ACTION
https://phab.mercurial-scm.org/D6927/new/
REVISION DETAIL
https://phab.mercurial-scm.org/D6927
AFFECTED FILES
contrib/ci/lambda_functions/ci.py
contrib/ci/lambda_functions/web.py
contrib/ci/terraform/job_executor.tf
CHANGE DETAILS
diff --git a/contrib/ci/terraform/job_executor.tf b/contrib/ci/terraform/job_executor.tf
--- a/contrib/ci/terraform/job_executor.tf
+++ b/contrib/ci/terraform/job_executor.tf
@@ -142,6 +142,7 @@
"ec2:CreateTags",
"ec2:DescribeInstanceAttribute",
"ec2:DescribeInstances",
+ "ec2:DescribeSpotInstanceRequests",
]
resources = ["*"]
}
diff --git a/contrib/ci/lambda_functions/web.py b/contrib/ci/lambda_functions/web.py
--- a/contrib/ci/lambda_functions/web.py
+++ b/contrib/ci/lambda_functions/web.py
@@ -115,6 +115,7 @@
'<th>Scheduled At</th>',
'<th>Start Delay</th>',
'<th>Execution Time</th>',
+ '<th>Cost</th>',
'<th>Total Tests</th>',
'<th>Passed</th>',
'<th>Failed</th>',
@@ -136,14 +137,28 @@
start_time = datetime.datetime.utcfromtimestamp(job_info['start_time'])
start_delay = '%ds' % (start_time - schedule_time).total_seconds()
else:
+ start_time = None
start_delay = 'n/a'
if 'end_time' in job_info:
end_time = datetime.datetime.utcfromtimestamp(job_info['end_time'])
execution_time = '%ds' % (end_time - start_time).total_seconds()
+
+ instance_time = (end_time - start_time).total_seconds()
else:
execution_time = 'n/a'
+ if start_time is not None:
+ instance_time = (datetime.datetime.utcnow() - start_time).total_seconds()
+ else:
+ instance_time = None
+
+ if 'instance_hourly_cost' in job_info and instance_time is not None:
+ total_cost = float(job_info['instance_hourly_cost'] )/ 3600.0 * instance_time
+ total_cost = '$%.3f' % total_cost
+ else:
+ total_cost = 'n/a'
+
if 'test_count' in job_info:
test_count = '%d' % job_info['test_count']
else:
@@ -207,6 +222,7 @@
'<td>%s</td>' % schedule_time.isoformat(),
'<td>%s</td>' % start_delay,
'<td>%s</td>' % execution_entry,
+ '<td>%s</td>' % e(total_cost),
'<td>%s</td>' % test_count,
'<td>%s</td>' % pass_count,
'<td>%s</td>' % fail_entry,
diff --git a/contrib/ci/lambda_functions/ci.py b/contrib/ci/lambda_functions/ci.py
--- a/contrib/ci/lambda_functions/ci.py
+++ b/contrib/ci/lambda_functions/ci.py
@@ -116,6 +116,7 @@
state = event['detail']['state']
print('received %s for %s' % (state, instance_id))
+ ec2_client = boto3.client('ec2')
ec2 = boto3.resource('ec2')
dynamodb = boto3.resource('dynamodb')
@@ -132,7 +133,7 @@
job_table = dynamodb.Table(os.environ['DYNAMODB_JOB_TABLE'])
- react_to_instance_state_change(job_table, instance, state)
+ react_to_instance_state_change(ec2_client, job_table, instance, state)
def handle_try_server_upload(event, context):
@@ -645,7 +646,7 @@
)
-def react_to_instance_state_change(job_table, instance, state):
+def react_to_instance_state_change(ec2, job_table, instance, state):
"""React to a CI worker instance state change."""
now = decimal.Decimal(time.time())
@@ -690,17 +691,32 @@
# New instance/job seen. Record that.
if state == 'pending':
print('recording running state for job %s' % job_id)
+
+ # Try to record the cost to running this instance.
+ hourly_cost = None
+
+ if instance.spot_instance_request_id:
+ spot_instance_requests = ec2.describe_spot_instance_requests(
+ SpotInstanceRequestIds=[instance.spot_instance_request_id],
+ )['SpotInstanceRequests']
+
+ if spot_instance_requests:
+ hourly_cost = decimal.Decimal(
+ spot_instance_requests[0]['ActualBlockHourlyPrice'])
+
job_table.update_item(
Key={'job_id': job_id},
UpdateExpression=(
'set execution_state = :state, '
'instance_id = :instance_id, '
+ 'instance_hourly_cost = :hourly_cost, '
'start_time = :start_time, '
'exit_clean = :exit_clean'
),
ExpressionAttributeValues={
':state': 'running',
':instance_id': instance.instance_id,
+ ':hourly_cost': hourly_cost,
':start_time': now,
':exit_clean': False,
},
To: indygreg, #hg-reviewers
Cc: mercurial-devel
More information about the Mercurial-devel
mailing list