Skip to content

Commit

Permalink
add restore from backup
Browse files Browse the repository at this point in the history
  • Loading branch information
maismail committed Jan 29, 2024
1 parent f6c6702 commit 588d24b
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 45 deletions.
89 changes: 48 additions & 41 deletions docs/resources/cluster_from_backup.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ resource "hopsworksai_cluster_from_backup" "cluster" {
- `autoscale` (Block List, Max: 1) Setup auto scaling. (see [below for nested schema](#nestedblock--autoscale))
- `aws_attributes` (Block List, Max: 1) The configurations required to run the cluster on Amazon AWS. (see [below for nested schema](#nestedblock--aws_attributes))
- `azure_attributes` (Block List, Max: 1) The configurations required to run the cluster on Microsoft Azure. (see [below for nested schema](#nestedblock--azure_attributes))
- `gcp_attributes` (Block List, Max: 1) The configurations required to run the cluster on Google GCP. (see [below for nested schema](#nestedblock--gcp_attributes))
- `name` (String) The name of the cluster, must be unique.
- `open_ports` (Block List, Max: 1) Open the required ports to communicate with one of the Hopsworks services. (see [below for nested schema](#nestedblock--open_ports))
- `ssh_key` (String) The ssh key name that will be attached to this cluster.
Expand All @@ -49,7 +50,6 @@ resource "hopsworksai_cluster_from_backup" "cluster" {
- `creation_date` (String) The creation date of the cluster. The date is represented in RFC3339 format.
- `custom_hosted_zone` (String) Override the default cloud.hopsworks.ai Hosted Zone. This option is available only to users with necessary privileges.
- `deactivate_hopsworksai_log_collection` (Boolean) Allow Hopsworks.ai to collect services logs to help diagnose issues with the cluster. By deactivating this option, you will not be able to get full support from our teams.
- `gcp_attributes` (List of Object) The configurations required to run the cluster on Google GCP. (see [below for nested schema](#nestedatt--gcp_attributes))
- `head` (List of Object) The configurations of the head node of the cluster. (see [below for nested schema](#nestedatt--head))
- `id` (String) The ID of this resource.
- `init_script` (String) A bash script that will run on all nodes during their initialization (must start with #!/usr/bin/env bash)
Expand Down Expand Up @@ -216,6 +216,53 @@ Read-Only:



<a id="nestedblock--gcp_attributes"></a>
### Nested Schema for `gcp_attributes`

Optional:

- `network` (Block List, Max: 1) The network configurations. (see [below for nested schema](#nestedblock--gcp_attributes--network))
- `service_account_email` (String) The service account email address that the cluster will be started with.

Read-Only:

- `bucket` (List of Object) The bucket configurations. (see [below for nested schema](#nestedatt--gcp_attributes--bucket))
- `disk_encryption` (List of Object) The disk encryption configuration. (see [below for nested schema](#nestedatt--gcp_attributes--disk_encryption))
- `gke_cluster_name` (String) The name of the Google GKE cluster.
- `project_id` (String) The GCP project where the cluster will be created.
- `region` (String) The GCP region where the cluster will be created.
- `zone` (String) The GCP region where the cluster will be created.

<a id="nestedblock--gcp_attributes--network"></a>
### Nested Schema for `gcp_attributes.network`

Required:

- `subnet_id` (String) The subnet id.
- `vpc_id` (String) The VPC id.

Optional:

- `security_group_id` (String) The security group id.


<a id="nestedatt--gcp_attributes--bucket"></a>
### Nested Schema for `gcp_attributes.bucket`

Read-Only:

- `name` (String)


<a id="nestedatt--gcp_attributes--disk_encryption"></a>
### Nested Schema for `gcp_attributes.disk_encryption`

Read-Only:

- `customer_managed_encryption_key` (String)



<a id="nestedblock--open_ports"></a>
### Nested Schema for `open_ports`

Expand Down Expand Up @@ -265,46 +312,6 @@ Optional:



<a id="nestedatt--gcp_attributes"></a>
### Nested Schema for `gcp_attributes`

Read-Only:

- `bucket` (List of Object) (see [below for nested schema](#nestedobjatt--gcp_attributes--bucket))
- `disk_encryption` (List of Object) (see [below for nested schema](#nestedobjatt--gcp_attributes--disk_encryption))
- `gke_cluster_name` (String)
- `network` (List of Object) (see [below for nested schema](#nestedobjatt--gcp_attributes--network))
- `project_id` (String)
- `region` (String)
- `service_account_email` (String)
- `zone` (String)

<a id="nestedobjatt--gcp_attributes--bucket"></a>
### Nested Schema for `gcp_attributes.bucket`

Read-Only:

- `name` (String)


<a id="nestedobjatt--gcp_attributes--disk_encryption"></a>
### Nested Schema for `gcp_attributes.disk_encryption`

Read-Only:

- `customer_managed_encryption_key` (String)


<a id="nestedobjatt--gcp_attributes--network"></a>
### Nested Schema for `gcp_attributes.network`

Read-Only:

- `network_name` (String)
- `subnetwork_name` (String)



<a id="nestedatt--head"></a>
### Nested Schema for `head`

Expand Down
2 changes: 2 additions & 0 deletions hopsworksai/internal/api/apis.go
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ func NewClusterFromBackup(ctx context.Context, apiClient APIHandler, backupId st
tflog.Debug(ctx, fmt.Sprintf("restore aws cluster: #%v", createRequest))
case CreateAzureClusterFromBackup, *CreateAzureClusterFromBackup:
tflog.Debug(ctx, fmt.Sprintf("restore azure cluster: #%v", createRequest))
case CreateGCPClusterFromBackup, *CreateGCPClusterFromBackup:
tflog.Debug(ctx, fmt.Sprintf("restore gcp cluster: #%v", createRequest))

Check warning on line 253 in hopsworksai/internal/api/apis.go

View check run for this annotation

Codecov / codecov/patch

hopsworksai/internal/api/apis.go#L252-L253

Added lines #L252 - L253 were not covered by tests
default:
return "", fmt.Errorf("unknown create request #%v", createRequest)
}
Expand Down
7 changes: 7 additions & 0 deletions hopsworksai/internal/api/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,13 @@ type CreateAWSClusterFromBackup struct {
SecurityGroupId string `json:"securityGroupId,omitempty"`
}

type CreateGCPClusterFromBackup struct {
CreateClusterFromBackup
ServiceAccountEmail string `json:"serviceAccountEmail"`
NetworkName string `json:"networkName"`
SubNetworkName string `json:"subNetworkName"`
}

type NewClusterFromBackupRequest struct {
CreateRequest interface{} `json:"cluster"`
}
Expand Down
42 changes: 38 additions & 4 deletions hopsworksai/resource_cluster_from_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package hopsworksai

import (
"context"
"strings"
"time"

"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
Expand Down Expand Up @@ -37,7 +38,7 @@ func clusterFromBackupResource() *schema.Resource {
baseSchema["aws_attributes"].Optional = true
baseSchema["aws_attributes"].ForceNew = true
baseSchema["aws_attributes"].MaxItems = 1
baseSchema["aws_attributes"].ConflictsWith = []string{"azure_attributes"}
baseSchema["aws_attributes"].ConflictsWith = []string{"azure_attributes", "gcp_attributes"}

clusterAWSAttributesSchema := baseSchema["aws_attributes"].Elem.(*schema.Resource).Schema
clusterAWSAttributesSchema["instance_profile_arn"].Optional = true
Expand All @@ -50,11 +51,22 @@ func clusterFromBackupResource() *schema.Resource {
baseSchema["azure_attributes"].Optional = true
baseSchema["azure_attributes"].ForceNew = true
baseSchema["azure_attributes"].MaxItems = 1
baseSchema["azure_attributes"].ConflictsWith = []string{"aws_attributes"}
baseSchema["azure_attributes"].ConflictsWith = []string{"aws_attributes", "gcp_attributes"}

clusterAZUREAttributesSchema := baseSchema["azure_attributes"].Elem.(*schema.Resource).Schema
clusterAZUREAttributesSchema["network"] = azureAttributesSchema().Schema["network"]

// allow changing gcp
baseSchema["gcp_attributes"].Optional = true
baseSchema["gcp_attributes"].ForceNew = true
baseSchema["gcp_attributes"].MaxItems = 1
baseSchema["gcp_attributes"].ConflictsWith = []string{"aws_attributes", "azure_attributes"}

clusterGCPAttributesSchema := baseSchema["gcp_attributes"].Elem.(*schema.Resource).Schema
clusterGCPAttributesSchema["service_account_email"].Optional = true
clusterGCPAttributesSchema["service_account_email"].ForceNew = true
clusterGCPAttributesSchema["network"] = awsAttributesSchema().Schema["network"]

// allow the following attributes to be updated later after creation
baseSchema["update_state"] = clusterResourceSchema["update_state"]
baseSchema["open_ports"] = clusterResourceSchema["open_ports"]
Expand Down Expand Up @@ -129,6 +141,10 @@ func resourceClusterFromBackupCreate(ctx context.Context, d *schema.ResourceData
restoreRequest = &api.CreateAzureClusterFromBackup{
CreateClusterFromBackup: baseRequest,
}
case api.GCP:
restoreRequest = &api.CreateGCPClusterFromBackup{
CreateClusterFromBackup: baseRequest,
}

Check warning on line 147 in hopsworksai/resource_cluster_from_backup.go

View check run for this annotation

Codecov / codecov/patch

hopsworksai/resource_cluster_from_backup.go#L144-L147

Added lines #L144 - L147 were not covered by tests
default:
return diag.Errorf("Unknown cloud provider %s for backup %s", backup.CloudProvider, backupId)
}
Expand All @@ -155,7 +171,7 @@ func resourceClusterFromBackupCreate(ctx context.Context, d *schema.ResourceData
awsRequest.SecurityGroupId = v.(string)
}
} else {
return diag.Errorf("incompatible cloud configuration, expected azure_attributes instead")
return diag.Errorf("incompatible cloud configuration, expected %s_attributes instead", strings.ToLower(backup.CloudProvider.String()))
}
}

Expand All @@ -177,7 +193,25 @@ func resourceClusterFromBackupCreate(ctx context.Context, d *schema.ResourceData
azureRequest.SecurityGroupName = v.(string)
}
} else {
return diag.Errorf("incompatible cloud configuration, expected aws_attributes instead")
return diag.Errorf("incompatible cloud configuration, expected %s_attributes instead", strings.ToLower(backup.CloudProvider.String()))
}
}

if gcp, ok := d.GetOk("gcp_attributes"); ok && len(gcp.([]interface{})) > 0 {
if gcpRequest, okV := restoreRequest.(*api.CreateGCPClusterFromBackup); okV {
if v, ok := d.GetOk("gcp_attributes.0.service_account_email"); ok {
gcpRequest.ServiceAccountEmail = v.(string)
}

Check warning on line 204 in hopsworksai/resource_cluster_from_backup.go

View check run for this annotation

Codecov / codecov/patch

hopsworksai/resource_cluster_from_backup.go#L201-L204

Added lines #L201 - L204 were not covered by tests

if v, ok := d.GetOk("gcp_attributes.0.network.0.network_name"); ok {
gcpRequest.NetworkName = v.(string)
}

Check warning on line 208 in hopsworksai/resource_cluster_from_backup.go

View check run for this annotation

Codecov / codecov/patch

hopsworksai/resource_cluster_from_backup.go#L206-L208

Added lines #L206 - L208 were not covered by tests

if v, ok := d.GetOk("gcp_attributes.0.network.0.subnetwork_name"); ok {
gcpRequest.SubNetworkName = v.(string)
}
} else {
return diag.Errorf("incompatible cloud configuration, expected %s_attributes instead", strings.ToLower(backup.CloudProvider.String()))

Check warning on line 214 in hopsworksai/resource_cluster_from_backup.go

View check run for this annotation

Codecov / codecov/patch

hopsworksai/resource_cluster_from_backup.go#L210-L214

Added lines #L210 - L214 were not covered by tests
}
}

Expand Down

0 comments on commit 588d24b

Please sign in to comment.