From 00380df5a170ab3d49d67c7e8638a0e0118e6d5e Mon Sep 17 00:00:00 2001 From: Peter Ruan Date: Sat, 27 Jun 2026 08:42:51 -0700 Subject: [PATCH] add GCE resource summary to cloud_watch rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- config/config.yaml | 1 + lib/configuration.rb | 4 +- lib/launchers/amz.rb | 10 +- lib/launchers/gce.rb | 14 +++ tools/cloud_watch.rb | 43 +++++--- tools/resource_monitor.rb | 203 +++++++++++++++++++++++++++++++++++--- 6 files changed, 246 insertions(+), 29 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 71bb40cadd..ac9dbf3dd6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -215,6 +215,7 @@ services: network: default scopes: &gce_client_scopes - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/cloud-platform.read-only # json_cred: path/to/serviceaccount/json/creds/unless/using/signet/below host_opts: # user: provisioned machine username diff --git a/lib/configuration.rb b/lib/configuration.rb index ef7ddedb80..d11019710a 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -25,7 +25,7 @@ def initialize(opts = {}) end def load_file(config_file) - config = YAML.load_file(config_file) + config = YAML.unsafe_load_file(config_file) end ## return full raw configuration @@ -37,7 +37,7 @@ def raw # merge config from environment if present if ENV["BUSHSLICER_CONFIG"] && !ENV["BUSHSLICER_CONFIG"].strip.empty? - raw_configs << YAML.load(ENV["BUSHSLICER_CONFIG"]) + raw_configs << YAML.unsafe_load(ENV["BUSHSLICER_CONFIG"]) end # merge all config files diff --git a/lib/launchers/amz.rb b/lib/launchers/amz.rb index 732c4f916e..b2619710cf 100755 --- a/lib/launchers/amz.rb +++ b/lib/launchers/amz.rb @@ -1,5 +1,9 @@ #!/usr/bin/env ruby -require 'aws-sdk' +require 'aws-sdk-ec2' +require 'aws-sdk-iam' +require 'aws-sdk-route53' +require 'aws-sdk-s3' +require 'aws-sdk-sts' lib_path = File.expand_path(File.dirname(File.dirname(__FILE__))) unless $LOAD_PATH.any? {|p| File.expand_path(p) == lib_path} @@ -556,7 +560,9 @@ def get_volume_state(volume) # @return [Array ] def get_regions - client_ec2.describe_regions.to_a[0][0] + client_ec2.describe_regions( + filters: [{ name: 'opt-in-status', values: ['opt-in-not-required', 'opted-in'] }] + ).to_a[0][0] end def default_zone diff --git a/lib/launchers/gce.rb b/lib/launchers/gce.rb index a1fd58aed0..a46a8f0b21 100644 --- a/lib/launchers/gce.rb +++ b/lib/launchers/gce.rb @@ -1,4 +1,5 @@ require 'google/apis/compute_v1' +require 'google/apis/cloudresourcemanager_v1' require 'googleauth' # require 'signet/oauth_2/client' @@ -78,6 +79,19 @@ def compute return @compute end + def crm + return @crm if @crm + + @crm = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + @crm.authorization = compute.authorization + return @crm + end + + def get_iam_policy(project: nil) + project ||= @config[:project] + crm.get_project_iam_policy(project) + end + # image_name is RE2 syntax see: # https://github.com/google/re2/blob/master/doc/syntax.txt # @return [Google::Apis::ComputeV1::Image] single image, diff --git a/tools/cloud_watch.rb b/tools/cloud_watch.rb index 9c3f30c5c5..c03a5ec9c5 100755 --- a/tools/cloud_watch.rb +++ b/tools/cloud_watch.rb @@ -1,27 +1,23 @@ #!/usr/bin/env ruby -$LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../lib") -$LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../tools") ''" Helper utility to interact with AWS services on the command-line "'' -require 'commander' +# Fast path for --version/--help before loading heavy deps +if ARGV == ['--version'] + puts "CloudWatch 0.0.1" + exit +end -require 'launchers/amz' +$LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../lib") +$LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../tools") +require 'commander' require 'collections' require 'common' -require 'cucuhttp' -require 'jenkins_api_client' require 'text-table' -# GCE specific -require 'google/apis/compute_v1' -require 'googleauth' -require 'launchers/openstack' - -# user libs -require 'resource_monitor' +require 'launchers/cloud_helper' @@ -51,6 +47,10 @@ def run c.syntax = "#{File.basename __FILE__} -r [--all]" c.description = 'display resource summary for AWS' c.action do |args, options| + require 'launchers/amz' + require 'cucuhttp' + require 'jenkins_api_client' + require 'resource_monitor' ps = AwsResources.new options.config = conf say 'Getting summary...' @@ -62,6 +62,9 @@ def run c.syntax = "#{File.basename __FILE__}" c.description = 'display resource summary for Openstack' c.action do |args, options| + require 'launchers/openstack' + require 'cucuhttp' + require 'resource_monitor' ps = OpenstackResources.new options.config = conf say 'Getting summary...' @@ -69,6 +72,20 @@ def run end end + command :gce do |c| + c.syntax = "#{File.basename __FILE__}" + c.description = 'display resource summary for GCE' + c.action do |args, options| + require 'launchers/gce' + require 'cucuhttp' + require 'resource_monitor' + ps = GceResources.new + options.config = conf + say 'Getting summary...' + ps.summarize_resources + end + end + run! end end diff --git a/tools/resource_monitor.rb b/tools/resource_monitor.rb index 8fc7b2bad1..bdc73fd287 100644 --- a/tools/resource_monitor.rb +++ b/tools/resource_monitor.rb @@ -1,3 +1,4 @@ +require 'set' require 'text-table' require 'thread' require 'openshift_qe_slack' @@ -75,15 +76,21 @@ def get_vpcs(global_region: :"AWS-CLOUD-USAGE") region_names = regions.map {|r| r.region_name } vpcs_hash = {} raw_vpcs = [] + mutex = Mutex.new threads = [] - ## XXX commnet out thread implmentation for now as it's flaky when when in jenkins regions.each do | region | - threads << Thread.new(Amz_EC2.new(region: region.region_name)) do |aws| - aws = Amz_EC2.new(service_name: global_region, region: region.region_name) - vpcs_in_region = aws.get_vpcs() - raw_vpcs << vpcs_in_region - if vpcs_in_region.count > 0 - vpcs_hash[region.region_name] = vpcs_in_region + threads << Thread.new(region.region_name) do |region_name| + begin + aws = Amz_EC2.new(service_name: global_region, region: region_name) + vpcs_in_region = aws.get_vpcs() + mutex.synchronize do + raw_vpcs << vpcs_in_region + if vpcs_in_region.count > 0 + vpcs_hash[region.region_name] = vpcs_in_region + end + end + rescue => e + puts "WARNING: failed to get VPCs for region #{region_name}: #{e.message}" end end end @@ -155,18 +162,190 @@ def summarize_resources(resources: ['s3, vpc']) end - class GceRources < ResourceMonitor + class GceResources < ResourceMonitor attr_accessor :gce def initialize(jenkins: nil) @gce = GCE.new + @resource_limits = { + :instances => 1000, + :cpus => 5000, + :disks => 2000, + :networks => 100 + } end - # @return - def summarize_resource() - summary = [] + # @return Array of networks (VPCs) + def get_networks gce = @gce project = gce.config[:project] + compute = gce.compute + networks = compute.list_networks(project) + return networks.items || [] + end + + # @return Array of [name, creation_timestamp, subnet_count] + def extract_networks_data(networks) + data = [] + networks.each do |network| + subnet_count = network.subnetworks ? network.subnetworks.count : 0 + row_data = [ + network.name, + network.creation_timestamp || 'N/A', + subnet_count + ] + data << row_data + end + return data + end + + ## print out summary in a text table format + def print_summary(data_rows, headers) + table = Text::Table.new + + table.head = headers + data_rows.each do |data| + table.rows << data + end + puts table + end + + # @return Hash of instances keyed by zone + def get_instances_by_zone + gce = @gce + project = gce.config[:project] + regions = gce.regions + instances_hash = {} + raw_instances = [] + mutex = Mutex.new + threads = [] + + regions.each do |region, zones| + zones.each do |zone| + threads << Thread.new(zone) do |zone_name| + begin + instances_in_zone = gce.get_instances_by_status(zone: zone_name, status: 'RUNNING') + mutex.synchronize do + raw_instances << instances_in_zone + if instances_in_zone && instances_in_zone.count > 0 + instances_hash[zone_name] = instances_in_zone + end + end + rescue => e + puts "WARNING: failed to get instances for zone #{zone_name}: #{e.message}" + end + end + end + end + threads.each(&:join) + return instances_hash, raw_instances + end + + # @return Array of [name, status, creation_date, machine_type] + def extract_instances_data(raw_instances) + instances = raw_instances.flatten.compact + data = [] + instances.each do |inst| + row_data = [ + inst.name, + inst.status, + inst.creation_timestamp, + inst.machine_type.split('/')[-1] + ] + data << row_data + end + return data + end + + # @return summary of GCE resources + def summarize_resources(resources: []) + instances_hash, raw_instances = self.get_instances_by_zone + instances_total = 0 + summary_data = [] + limits_msgs = [] + + instances_hash.each do |zone, instances| + instances_total += instances.count + row_data = [zone, instances.count] + summary_data << row_data + limits_msg = over_limit?(resource_type: "instances in zone #{zone}", + resource_value: instances.count, + resource_limit: @resource_limits[:instances], + percentage: 80) + limits_msgs << limits_msg unless limits_msg.nil? + end + + zone_header = ['zone', 'total_instances'] + print_summary(summary_data, zone_header) + + instances_data = extract_instances_data(raw_instances) + instances_header = ['name', 'status', 'creation_date', 'machine_type'] + print_summary(instances_data, instances_header) + + instances_limits_msg = over_limit?(resource_type: "total instances", + resource_value: instances_total, + resource_limit: @resource_limits[:instances], + percentage: 90) + limits_msgs << instances_limits_msg unless instances_limits_msg.nil? + + # Get and display networks (VPCs) + networks = self.get_networks + networks_data = extract_networks_data(networks) + networks_header = ['name', 'creation_date', 'subnet_count'] + print_summary(networks_data, networks_header) + + networks_limits_msg = over_limit?(resource_type: "total networks (VPCs)", + resource_value: networks.count, + resource_limit: @resource_limits[:networks], + percentage: 90) + limits_msgs << networks_limits_msg unless networks_limits_msg.nil? + + # IAM policy member breakdown + begin + iam_summary = get_iam_policy_summary + + iam_role_data = iam_summary[:by_role].map { |role, members| [role, members.count] } + print_summary(iam_role_data, ['role', 'member_count']) + + iam_type_data = iam_summary[:by_type].map { |type, count| [type, count] } + print_summary(iam_type_data, ['member_type', 'count']) + rescue => e + puts "WARNING: failed to get IAM policy: #{e.message}" + iam_summary = nil + end + + print("#" * 70 + "\n") + msg_header = "# GCE usage summary\n" + print("#{msg_header}") + print("#" * 70 + "\n") + print("Total instances: #{instances_total}/#{@resource_limits[:instances]}\n") + print("Total networks (VPCs): #{networks.count}/#{@resource_limits[:networks]}\n") + if iam_summary + print("Total unique IAM members: #{iam_summary[:total_unique]}\n") + end + + limits_msgs.unshift(msg_header) if limits_msgs.count > 0 + notify_limits(limits_msgs) + end + + def get_iam_policy_summary + policy = @gce.get_iam_policy + by_role = {} + all_members = Set.new + + policy.bindings.each do |binding| + role_short = binding.role.split('/').last + by_role[role_short] = binding.members + binding.members.each { |m| all_members.add(m) } + end + + by_type = Hash.new(0) + all_members.each do |member| + type = member.split(':').first + by_type[type] += 1 + end + + { by_role: by_role, by_type: by_type, total_unique: all_members.count } end end @@ -178,7 +357,7 @@ def initialize(jenkins: nil) @azure = Azure.new end - def summarize_resource(cm) + def summarize_resources(cm) print "Summaring resource ..." end