D6927: ci: report cost to run each job

[indygreg (Gregory Szorc)]

indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  The spot instance request contains details on the cost to run
  the instance. Let's record the hourly cost to run an instance
  in DynamoDB so we can use it to calculate the total compute cost
  to run a single job.
  
  And now that we store it, expose it in the web interface so there
  is visibility.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D6927

AFFECTED FILES
  contrib/ci/lambda_functions/ci.py
  contrib/ci/lambda_functions/web.py
  contrib/ci/terraform/job_executor.tf

CHANGE DETAILS

diff --git a/contrib/ci/terraform/job_executor.tf b/contrib/ci/terraform/job_executor.tf
--- a/contrib/ci/terraform/job_executor.tf
+++ b/contrib/ci/terraform/job_executor.tf
@@ -142,6 +142,7 @@
       "ec2:CreateTags",
       "ec2:DescribeInstanceAttribute",
       "ec2:DescribeInstances",
+      "ec2:DescribeSpotInstanceRequests",
     ]
     resources = ["*"]
   }
diff --git a/contrib/ci/lambda_functions/web.py b/contrib/ci/lambda_functions/web.py
--- a/contrib/ci/lambda_functions/web.py
+++ b/contrib/ci/lambda_functions/web.py
@@ -115,6 +115,7 @@
                 '<th>Scheduled At</th>',
                 '<th>Start Delay</th>',
                 '<th>Execution Time</th>',
+                '<th>Cost</th>',
                 '<th>Total Tests</th>',
                 '<th>Passed</th>',
                 '<th>Failed</th>',
@@ -136,14 +137,28 @@
                         start_time = datetime.datetime.utcfromtimestamp(job_info['start_time'])
                         start_delay = '%ds' % (start_time - schedule_time).total_seconds()
                     else:
+                        start_time = None
                         start_delay = 'n/a'
 
                     if 'end_time' in job_info:
                         end_time = datetime.datetime.utcfromtimestamp(job_info['end_time'])
                         execution_time = '%ds' %  (end_time - start_time).total_seconds()
+
+                        instance_time = (end_time - start_time).total_seconds()
                     else:
                         execution_time = 'n/a'
 
+                        if start_time is not None:
+                            instance_time = (datetime.datetime.utcnow() - start_time).total_seconds()
+                        else:
+                            instance_time = None
+
+                    if 'instance_hourly_cost' in job_info and instance_time is not None:
+                        total_cost = float(job_info['instance_hourly_cost'] )/ 3600.0 * instance_time
+                        total_cost = '$%.3f' % total_cost
+                    else:
+                        total_cost = 'n/a'
+
                     if 'test_count' in job_info:
                         test_count = '%d' % job_info['test_count']
                     else:
@@ -207,6 +222,7 @@
                         '<td>%s</td>' % schedule_time.isoformat(),
                         '<td>%s</td>' % start_delay,
                         '<td>%s</td>' % execution_entry,
+                        '<td>%s</td>' % e(total_cost),
                         '<td>%s</td>' % test_count,
                         '<td>%s</td>' % pass_count,
                         '<td>%s</td>' % fail_entry,
diff --git a/contrib/ci/lambda_functions/ci.py b/contrib/ci/lambda_functions/ci.py
--- a/contrib/ci/lambda_functions/ci.py
+++ b/contrib/ci/lambda_functions/ci.py
@@ -116,6 +116,7 @@
     state = event['detail']['state']
     print('received %s for %s' % (state, instance_id))
 
+    ec2_client = boto3.client('ec2')
     ec2 = boto3.resource('ec2')
     dynamodb = boto3.resource('dynamodb')
 
@@ -132,7 +133,7 @@
 
     job_table = dynamodb.Table(os.environ['DYNAMODB_JOB_TABLE'])
 
-    react_to_instance_state_change(job_table, instance, state)
+    react_to_instance_state_change(ec2_client, job_table, instance, state)
 
 
 def handle_try_server_upload(event, context):
@@ -644,7 +645,7 @@
             )
 
 
-def react_to_instance_state_change(job_table, instance, state):
+def react_to_instance_state_change(ec2, job_table, instance, state):
     """React to a CI worker instance state change."""
     now = decimal.Decimal(time.time())
 
@@ -689,17 +690,32 @@
     # New instance/job seen. Record that.
     if state == 'pending':
         print('recording running state for job %s' % job_id)
+
+        # Try to record the cost to running this instance.
+        hourly_cost = None
+
+        if instance.spot_instance_request_id:
+            spot_instance_requests = ec2.describe_spot_instance_requests(
+                SpotInstanceRequestIds=[instance.spot_instance_request_id],
+            )['SpotInstanceRequests']
+
+            if spot_instance_requests:
+                hourly_cost = decimal.Decimal(
+                    spot_instance_requests[0]['ActualBlockHourlyPrice'])
+
         job_table.update_item(
             Key={'job_id': job_id},
             UpdateExpression=(
                 'set execution_state = :state, '
                 'instance_id = :instance_id, '
+                'instance_hourly_cost = :hourly_cost, '
                 'start_time = :start_time, '
                 'exit_clean = :exit_clean'
             ),
             ExpressionAttributeValues={
                 ':state': 'running',
                 ':instance_id': instance.instance_id,
+                ':hourly_cost': hourly_cost,
                 ':start_time': now,
                 ':exit_clean': False,
             },



To: indygreg, #hg-reviewers
Cc: mercurial-devel