[GCP] Get disk size from image

.github/workflows/go.yml

@@ -1,6 +1,3 @@
-# This workflow will build a golang project
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
-
 name: Go
 
 on:
@@ -9,13 +6,28 @@ on:
   pull_request:
     branches: [ "main" ]
 
+
 jobs:
 
   build:
     runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      id-token: write
+
     steps:
     - uses: actions/checkout@v3
 
+    # Configure Workload Identity Federation via a credentials file.
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v1'
+      with:
+        workload_identity_provider: 'projects/536185737408/locations/global/workloadIdentityPools/carbonifer-gh-pool/providers/carbonifer-oidc-provider'
+        service_account: '[email protected]'
+      if: github.event_name != 'pull_request'
+
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
@@ -37,5 +49,13 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
 
-    - name: Test
-      run: go test -v ./... 
+    - name: Test with credentials
+      run: go test -v ./...
+      if: github.event_name != 'pull_request'
+
+    # If run from fork, we should not use tests requiring credentials
+    - name: Test without credentials
+      run: go test -v ./...
+      env:
+        SKIP_WITH_CREDENTIALS: true
+      if: github.event_name == 'pull_request'

.gitignore

@@ -21,7 +21,7 @@ go.work
 .DS_Store
 
 # Carbonifer
-**/.carbonifer*/*
+**/.carbonifer*
 
 # Local .terraform directories
 **/.terraform/*
@@ -36,3 +36,9 @@ crash.log
 crash.*.log
 
 **/__debug_bin
+
+# vscode
+.vscode
+
+# Ignore generated credentials from google-github-actions/auth
+gha-creds-*.json

cmd/plan.go

@@ -46,7 +46,10 @@ var planCmd = &cobra.Command{
 		log.Debugf("Workdir : %v", workdir)
 
 		// Read resources from terraform plan
-		resources := terraform.GetResources()
+		resources, err := terraform.GetResources()
+		if err != nil {
+			log.Fatal(err)
+		}
 
 		// Estimate CO2 emissions
 		estimations := estimate.EstimateResources(resources)

internal/estimate/estimate.go

@@ -24,7 +24,7 @@ func EstimateResources(resourceList []resources.Resource) EstimationReport {
 	for _, resource := range resourceList {
 		estimationResource, uerr := EstimateResource(resource)
 		if uerr != nil {
-			logrus.Warnf("Skipping unsupported provider %v: %v.%v", uerr.Provider, resource.GetIndentification().ResourceType, resource.GetIndentification().Name)
+			logrus.Warnf("Skipping unsupported provider %v: %v.%v", uerr.Provider, resource.GetIdentification().ResourceType, resource.GetIdentification().Name)
 		}
 
 		if resource.IsSupported() {
@@ -56,11 +56,11 @@ func EstimateResource(resource resources.Resource) (*EstimationResource, *provid
 	if !resource.IsSupported() {
 		return estimateNotSupported(resource.(resources.UnsupportedResource)), nil
 	}
-	switch resource.GetIndentification().Provider {
+	switch resource.GetIdentification().Provider {
 	case providers.GCP:
 		return estimateGCP(resource), nil
 	default:
-		return nil, &providers.UnsupportedProviderError{Provider: resource.GetIndentification().Provider.String()}
+		return nil, &providers.UnsupportedProviderError{Provider: resource.GetIdentification().Provider.String()}
 	}
 }
 
@@ -80,7 +80,7 @@ func estimateGCP(resource resources.Resource) *EstimationResource {
 	}
 
 	// Regional grid emission per unit of time
-	regionEmissions, err := GCPRegionEmission(resource.GetIndentification().Region) // gCO2eq /kWh
+	regionEmissions, err := GCPRegionEmission(resource.GetIdentification().Region) // gCO2eq /kWh
 	if err != nil {
 		log.Fatalf("Error while getting region emissions for %v: %v", resource.GetAddress(), err)
 	}

internal/estimate/estimate_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 var resourceGCPComputeBasic = resources.ComputeResource{
-	Identification: &resources.ComputeResourceIdentification{
+	Identification: &resources.ResourceIdentification{
 		Name:         "machine-name-1",
 		ResourceType: "type-1",
 		Provider:     providers.GCP,
@@ -26,7 +26,7 @@ var resourceGCPComputeBasic = resources.ComputeResource{
 }
 
 var resourceGCPComputeCPUType = resources.ComputeResource{
-	Identification: &resources.ComputeResourceIdentification{
+	Identification: &resources.ResourceIdentification{
 		Name:         "machine-name-2",
 		ResourceType: "type-1",
 		Provider:     providers.GCP,
@@ -42,7 +42,7 @@ var resourceGCPComputeCPUType = resources.ComputeResource{
 }
 
 var resourceAWSComputeBasic = resources.ComputeResource{
-	Identification: &resources.ComputeResourceIdentification{
+	Identification: &resources.ResourceIdentification{
 		Name:         "machine-name-1",
 		ResourceType: "type-1",
 		Provider:     providers.AWS,

internal/output/text.go

@@ -19,16 +19,16 @@ func GenerateReportText(report estimate.EstimationReport) string {
 
 	for _, resource := range report.Resources {
 		table.Append([]string{
-			resource.Resource.GetIndentification().ResourceType,
-			resource.Resource.GetIndentification().Name,
+			resource.Resource.GetIdentification().ResourceType,
+			resource.Resource.GetIdentification().Name,
 			fmt.Sprintf(" %v %v", resource.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime),
 		})
 	}
 
 	for _, resource := range report.UnsupportedResources {
 		table.Append([]string{
-			resource.GetIndentification().ResourceType,
-			resource.GetIndentification().Name,
+			resource.GetIdentification().ResourceType,
+			resource.GetIdentification().Name,
 			"unsupported",
 		})
 	}

internal/resources/compute.go

@@ -17,49 +17,50 @@ type ComputeResourceSpecs struct {
 	ReplicationFactor int32
 }
 
-type ComputeResourceIdentification struct {
+type ResourceIdentification struct {
 	// Indentification
 	Name         string
 	ResourceType string
 	Provider     providers.Provider
 	Region       string
+	SelfLink     string
 }
 
 type ComputeResource struct {
-	Identification *ComputeResourceIdentification
+	Identification *ResourceIdentification
 	Specs          *ComputeResourceSpecs
 }
 
 func (r ComputeResource) IsSupported() bool {
 	return true
 }
 
-func (r ComputeResource) GetIndentification() *ComputeResourceIdentification {
+func (r ComputeResource) GetIdentification() *ResourceIdentification {
 	return r.Identification
 }
 
 func (r ComputeResource) GetAddress() string {
-	return fmt.Sprintf("%v.%v", r.GetIndentification().ResourceType, r.GetIndentification().Name)
+	return fmt.Sprintf("%v.%v", r.GetIdentification().ResourceType, r.GetIdentification().Name)
 }
 
 type UnsupportedResource struct {
-	Identification *ComputeResourceIdentification
+	Identification *ResourceIdentification
 }
 
 func (r UnsupportedResource) IsSupported() bool {
 	return false
 }
 
-func (r UnsupportedResource) GetIndentification() *ComputeResourceIdentification {
+func (r UnsupportedResource) GetIdentification() *ResourceIdentification {
 	return r.Identification
 }
 
 func (r UnsupportedResource) GetAddress() string {
-	return fmt.Sprintf("%v.%v", r.GetIndentification().ResourceType, r.GetIndentification().Name)
+	return fmt.Sprintf("%v.%v", r.GetIdentification().ResourceType, r.GetIdentification().Name)
 }
 
 type Resource interface {
 	IsSupported() bool
-	GetIndentification() *ComputeResourceIdentification
+	GetIdentification() *ResourceIdentification
 	GetAddress() string
 }

internal/resources/data.go

@@ -0,0 +1,30 @@
+package resources
+
+import "fmt"
+
+type DataImageSpecs struct {
+	DiskSizeGb float64
+}
+
+type DataImageResource struct {
+	Identification *ResourceIdentification
+	DataImageSpecs *DataImageSpecs
+}
+
+func (r DataImageResource) GetIdentification() *ResourceIdentification {
+	return r.Identification
+}
+
+func (r DataImageResource) GetAddress() string {
+	return fmt.Sprintf("data.%v.%v", r.GetIdentification().ResourceType, r.GetIdentification().Name)
+}
+
+func (r DataImageResource) GetKey() string {
+	return r.GetIdentification().SelfLink
+}
+
+type DataResource interface {
+	GetIdentification() *ResourceIdentification
+	GetAddress() string
+	GetKey() string
+}

internal/terraform/GCP.go

@@ -12,18 +12,18 @@ import (
 	"github.com/spf13/viper"
 )
 
-func GetResource(tfResource tfjson.StateResource) resources.Resource {
+func GetResource(tfResource tfjson.StateResource, dataResources *map[string]resources.DataResource) resources.Resource {
 	resourceId := getResourceIdentification(tfResource)
 	if resourceId.ResourceType == "google_compute_instance" {
-		specs := getComputeResourceSpecs(tfResource)
+		specs := getComputeResourceSpecs(tfResource, dataResources)
 		return resources.ComputeResource{
 			Identification: resourceId,
 			Specs:          specs,
 		}
 	}
 	if resourceId.ResourceType == "google_compute_disk" ||
 		resourceId.ResourceType == "google_compute_region_disk" {
-		specs := getComputeDiskResourceSpecs(tfResource)
+		specs := getComputeDiskResourceSpecs(tfResource, dataResources)
 		return resources.ComputeResource{
 			Identification: resourceId,
 			Specs:          specs,
@@ -34,7 +34,7 @@ func GetResource(tfResource tfjson.StateResource) resources.Resource {
 	}
 }
 
-func getResourceIdentification(resource tfjson.StateResource) *resources.ComputeResourceIdentification {
+func getResourceIdentification(resource tfjson.StateResource) *resources.ResourceIdentification {
 	region := resource.AttributeValues["region"]
 	if region == nil {
 		if resource.AttributeValues["zone"] != nil {
@@ -48,16 +48,24 @@ func getResourceIdentification(resource tfjson.StateResource) *resources.Compute
 			region = ""
 		}
 	}
+	selfLink := ""
+	if resource.AttributeValues["self_link"] != nil {
+		selfLink = resource.AttributeValues["self_link"].(string)
+	}
 
-	return &resources.ComputeResourceIdentification{
+	return &resources.ResourceIdentification{
 		Name:         resource.Name,
 		ResourceType: resource.Type,
 		Provider:     providers.GCP,
 		Region:       fmt.Sprint(region),
+		SelfLink:     selfLink,
 	}
 }
 
-func getComputeResourceSpecs(resource tfjson.StateResource) *resources.ComputeResourceSpecs {
+func getComputeResourceSpecs(
+	resource tfjson.StateResource,
+	dataResources *map[string]resources.DataResource) *resources.ComputeResourceSpecs {
+
 	machine_type := resource.AttributeValues["machine_type"].(string)
 	zone := resource.AttributeValues["zone"].(string)
 	machineType := providers.GetGCPMachineType(machine_type, zone)
@@ -69,7 +77,7 @@ func getComputeResourceSpecs(resource tfjson.StateResource) *resources.ComputeRe
 	var disks []disk
 	bootDisks := resource.AttributeValues["boot_disk"].([]interface{})
 	for _, bootDiskBlock := range bootDisks {
-		bootDisk := getBootDisk(resource.Address, bootDiskBlock.(map[string]interface{}))
+		bootDisk := getBootDisk(resource.Address, bootDiskBlock.(map[string]interface{}), dataResources)
 		disks = append(disks, bootDisk)
 	}
 
@@ -99,8 +107,11 @@ func getComputeResourceSpecs(resource tfjson.StateResource) *resources.ComputeRe
 	}
 }
 
-func getComputeDiskResourceSpecs(resource tfjson.StateResource) *resources.ComputeResourceSpecs {
-	disk := getDisk(resource.Address, resource.AttributeValues, false)
+func getComputeDiskResourceSpecs(
+	resource tfjson.StateResource,
+	dataResources *map[string]resources.DataResource) *resources.ComputeResourceSpecs {
+
+	disk := getDisk(resource.Address, resource.AttributeValues, false, dataResources)
 	hddSize := decimal.Zero
 	ssdSize := decimal.Zero
 	if disk.isSSD {
@@ -121,18 +132,18 @@ type disk struct {
 	replicationFactor int32
 }
 
-func getBootDisk(resourceAddress string, bootDiskBlock map[string]interface{}) disk {
+func getBootDisk(resourceAddress string, bootDiskBlock map[string]interface{}, dataResources *map[string]resources.DataResource) disk {
 	var disk disk
 	initParams := bootDiskBlock["initialize_params"]
 	for _, iP := range initParams.([]interface{}) {
 		initParam := iP.(map[string]interface{})
-		disk = getDisk(resourceAddress, initParam, true)
+		disk = getDisk(resourceAddress, initParam, true, dataResources)
 
 	}
 	return disk
 }
 
-func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDisk bool) disk {
+func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDisk bool, dataResources *map[string]resources.DataResource) disk {
 	disk := disk{
 		sizeGb:            viper.GetFloat64("provider.gcp.boot_disk.size"),
 		isSSD:             true,
@@ -159,8 +170,19 @@ func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDis
 		} else {
 			disk.sizeGb = viper.GetFloat64("provider.gcp.disk.size")
 		}
-		log.Warningf("%v : Boot disk size not declared. Please set it! (otherwise we set it to 10gb) ", resourceAddress)
-		// TODO if size not provided we can try to get it from image
+		diskImageLink, ok := diskBlock["image"]
+		if ok {
+			image, ok := (*dataResources)[diskImageLink.(string)]
+			if ok {
+				disk.sizeGb = (image.(resources.DataImageResource)).DataImageSpecs.DiskSizeGb
+			} else {
+				log.Warningf("%v : Disk image does not have a size declared, considering it default to be 10Gb ", resourceAddress)
+			}
+		} else {
+			log.Warningf("%v : Boot disk size not declared. Please set it! (otherwise we assume 10gb) ", resourceAddress)
+
+		}
+
 	}
 
 	replicaZones := diskBlock["replica_zones"]