Skip to content

Commit caa45bb

Browse files
0x2b3bfa0restyled-commitsDavidGOrtega
authored
Task status, heavy write fix and aws ips (#322)
* Add task script status reporting logic * Update task/common/machine/script.go * Add support for status reporting * Reduces S3 writes * Restyled by gofmt * Sort rclone on startup script * Fix IP address propagation :sweat_smile: It happened because DescribeInstances with InstanceIds = [] was the same as describing every imaginable instance in the account. * Remove dead code Co-authored-by: Restyled.io <[email protected]> Co-authored-by: DavidGOrtega <[email protected]>
1 parent d242ce5 commit caa45bb

File tree

14 files changed

+122
-54
lines changed

14 files changed

+122
-54
lines changed

‎iterative/resource_task.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,11 @@ func resourceTaskRead(ctx context.Context, d *schema.ResourceData, m interface{}
202202
}
203203
d.Set("events", events)
204204

205-
d.Set("status", task.Status(ctx))
205+
status, err := task.Status(ctx)
206+
if err != nil {
207+
return diagnostic(diags, err, diag.Warning)
208+
}
209+
d.Set("status", status)
206210

207211
logs, err := task.Logs(ctx)
208212
if err != nil {

‎task/aws/resources/resource_auto_scaling_group.go‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -113,28 +113,32 @@ func (a *AutoScalingGroup) Read(ctx context.Context) error {
113113
return common.NotFoundError
114114
}
115115

116-
var instancesInput ec2.DescribeInstancesInput
117-
for _, instance := range groups.AutoScalingGroups[0].Instances {
118-
instancesInput.InstanceIds = append(instancesInput.InstanceIds, aws.ToString(instance.InstanceId))
119-
}
120-
121116
a.Attributes.Addresses = []net.IP{}
122-
a.Attributes.Status = common.Status{common.StatusCodeRunning: 0}
123-
for instancesPaginator := ec2.NewDescribeInstancesPaginator(a.Client.Services.EC2, &instancesInput); instancesPaginator.HasMorePages(); {
124-
page, err := instancesPaginator.NextPage(ctx)
125-
if err != nil {
126-
return err
117+
a.Attributes.Status = common.Status{common.StatusCodeActive: 0}
118+
if len(groups.AutoScalingGroups[0].Instances) > 0 {
119+
var instancesInput ec2.DescribeInstancesInput
120+
for _, instance := range groups.AutoScalingGroups[0].Instances {
121+
instancesInput.InstanceIds = append(instancesInput.InstanceIds, aws.ToString(instance.InstanceId))
127122
}
128123

129-
for _, reservation := range page.Reservations {
130-
for _, instance := range reservation.Instances {
131-
status := string(instance.State.Name)
132-
if instance.StateReason != nil {
133-
status += " " + aws.ToString(instance.StateReason.Message)
134-
}
135-
a.Attributes.Status[common.StatusCode(status)]++
136-
if address := net.ParseIP(aws.ToString(instance.PublicIpAddress)); address != nil {
137-
a.Attributes.Addresses = append(a.Attributes.Addresses, address)
124+
for instancesPaginator := ec2.NewDescribeInstancesPaginator(a.Client.Services.EC2, &instancesInput); instancesPaginator.HasMorePages(); {
125+
page, err := instancesPaginator.NextPage(ctx)
126+
if err != nil {
127+
return err
128+
}
129+
130+
for _, reservation := range page.Reservations {
131+
for _, instance := range reservation.Instances {
132+
status := string(instance.State.Name)
133+
if instance.StateReason != nil {
134+
status += " " + aws.ToString(instance.StateReason.Message)
135+
}
136+
if status == "running" {
137+
a.Attributes.Status[common.StatusCodeActive]++
138+
}
139+
if address := net.ParseIP(aws.ToString(instance.PublicIpAddress)); address != nil {
140+
a.Attributes.Addresses = append(a.Attributes.Addresses, address)
141+
}
138142
}
139143
}
140144
}

‎task/aws/task.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -272,8 +272,12 @@ func (t *Task) Events(ctx context.Context) []common.Event {
272272
return t.Attributes.Events
273273
}
274274

275-
func (t *Task) Status(ctx context.Context) common.Status {
276-
return t.Attributes.Status
275+
func (t *Task) Status(ctx context.Context) (common.Status, error) {
276+
if err := t.Read(ctx); err != nil {
277+
return nil, err
278+
}
279+
280+
return machine.Status(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"], t.Attributes.Status)
277281
}
278282

279283
func (t *Task) GetKeyPair(ctx context.Context) (*ssh.DeterministicSSHKeyPair, error) {

‎task/az/resources/resource_virtual_machine_scale_set.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -224,17 +224,16 @@ func (v *VirtualMachineScaleSet) Read(ctx context.Context) error {
224224
}
225225

226226
v.Attributes.Events = []common.Event{}
227-
v.Attributes.Status = common.Status{common.StatusCodeRunning: 0}
227+
v.Attributes.Status = common.Status{common.StatusCodeActive: 0}
228228
scaleSetView, err := v.Client.Services.VirtualMachineScaleSets.GetInstanceView(ctx, v.Dependencies.ResourceGroup.Identifier, v.Identifier)
229229
if err != nil {
230230
return err
231231
}
232232
if scaleSetView.VirtualMachine.StatusesSummary != nil {
233233
for _, status := range *scaleSetView.VirtualMachine.StatusesSummary {
234234
code := to.String(status.Code)
235-
v.Attributes.Status[common.StatusCode(code)] = int(to.Int32(status.Count))
236235
if code == "ProvisioningState/succeeded" {
237-
v.Attributes.Status[common.StatusCodeRunning] = int(to.Int32(status.Count))
236+
v.Attributes.Status[common.StatusCodeActive] = int(to.Int32(status.Count))
238237
}
239238
}
240239
}

‎task/az/task.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,8 +265,12 @@ func (t *Task) Events(ctx context.Context) []common.Event {
265265
return t.Attributes.Events
266266
}
267267

268-
func (t *Task) Status(ctx context.Context) common.Status {
269-
return t.Attributes.Status
268+
func (t *Task) Status(ctx context.Context) (common.Status, error) {
269+
if err := t.Read(ctx); err != nil {
270+
return nil, err
271+
}
272+
273+
return machine.Status(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"], t.Attributes.Status)
270274
}
271275

272276
func (t *Task) GetKeyPair(ctx context.Context) (*ssh.DeterministicSSHKeyPair, error) {

‎task/common/machine/script.go‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,17 @@ while IFS= read -rd $'\0' variable; do
5050
export "$(perl -0777p -e 's/\\"/"/g;' -e 's/(.+?)="(.+)"/$1=$2/sg' <<< "$variable")"
5151
done < <(perl -0777pe 's/\n*(.+?=".*?((?<!\\)"|\\\\"))\n*/$1\x00/sg' /tmp/tpi-environment)
5252
53+
TPI_MACHINE_IDENTITY="$(uuidgen)"
54+
TPI_LOG_DIRECTORY="$(mktemp --directory)"
55+
TPI_DATA_DIRECTORY="/tmp/tpi-task"
56+
5357
sudo tee /etc/systemd/system/tpi-task.service > /dev/null <<END
5458
[Unit]
5559
After=default.target
5660
[Service]
5761
Type=simple
5862
ExecStart=/usr/bin/tpi-task
63+
ExecStop=/bin/bash -c 'systemctl is-system-running | grep stopping || echo "{\\\\"result\\\\": \\\\"\$SERVICE_RESULT\\\\", \\\\"code\\\\": \\\\"\$EXIT_STATUS\\\\", \\\\"status\\\\": \\\\"\$EXIT_CODE\\\\"}" > "$TPI_LOG_DIRECTORY/status-$TPI_MACHINE_IDENTITY" && rclone copy "$TPI_LOG_DIRECTORY" "$RCLONE_REMOTE/reports"'
5964
ExecStopPost=/usr/bin/tpi-task-shutdown
6065
Environment=HOME=/root
6166
EnvironmentFile=/tmp/tpi-environment
@@ -98,17 +103,20 @@ rclone copy "$RCLONE_REMOTE/data" /tmp/tpi-task
98103
sudo systemctl daemon-reload
99104
sudo systemctl enable tpi-task.service --now
100105
101-
TPI_MACHINE_IDENTITY="$(uuidgen)"
102-
TPI_LOG_DIRECTORY="$(mktemp --directory)"
103-
104106
while sleep 5; do
105107
journalctl > "$TPI_LOG_DIRECTORY/machine-$TPI_MACHINE_IDENTITY"
106108
journalctl --unit tpi-task > "$TPI_LOG_DIRECTORY/task-$TPI_MACHINE_IDENTITY"
107-
rclone copy "$TPI_LOG_DIRECTORY" "$RCLONE_REMOTE/log"
109+
if [[ "$(stat -t "$TPI_LOG_DIRECTORY")" != "$TPI_LOG_DIRECTORY_HASH" ]]; then
110+
TPI_LOG_DIRECTORY_HASH="$(md5sum "$TPI_LOG_DIRECTORY"/*)"
111+
rclone copy "$TPI_LOG_DIRECTORY" "$RCLONE_REMOTE/reports"
112+
fi
108113
done &
109114
110115
while sleep 10; do
111-
rclone copy /tmp/tpi-task "$RCLONE_REMOTE/data"
116+
if [[ "$(stat -t "$TPI_DATA_DIRECTORY")" != "$TPI_DATA_DIRECTORY_EPOCH" ]]; then
117+
TPI_DATA_DIRECTORY_EPOCH="$(find "$TPI_DATA_DIRECTORY" -printf "%%T@\n" | sort | tail -1)"
118+
rclone copy "$TPI_DATA_DIRECTORY" "$RCLONE_REMOTE/data"
119+
fi
112120
done &
113121
`,
114122
base64.StdEncoding.EncodeToString([]byte(script)),

‎task/common/machine/storage.go‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package machine
33
import (
44
"bytes"
55
"context"
6+
"encoding/json"
67
"io"
78
"path/filepath"
89
"strings"
@@ -14,23 +15,31 @@ import (
1415

1516
"github.com/rclone/rclone/fs"
1617
"github.com/rclone/rclone/fs/sync"
18+
19+
"terraform-provider-iterative/task/common"
1720
)
1821

19-
func Logs(ctx context.Context, remote string) ([]string, error) {
22+
type StatusReport struct {
23+
Result string
24+
Status string
25+
Code string
26+
}
27+
28+
func Reports(ctx context.Context, remote, prefix string) ([]string, error) {
2029
remoteFileSystem, err := fs.NewFs(ctx, remote)
2130
if err != nil {
2231
return nil, err
2332
}
2433

25-
entries, err := remoteFileSystem.List(ctx, "/log")
34+
entries, err := remoteFileSystem.List(ctx, "/reports")
2635
if err != nil {
2736
return nil, err
2837
}
2938

3039
var logs []string
3140
for _, entry := range entries {
3241
path := entry.Remote()
33-
if base := filepath.Base(path); !strings.HasPrefix(base, "task-") {
42+
if base := filepath.Base(path); !strings.HasPrefix(base, prefix+"-") {
3443
continue
3544
}
3645

@@ -53,6 +62,32 @@ func Logs(ctx context.Context, remote string) ([]string, error) {
5362
return logs, nil
5463
}
5564

65+
func Logs(ctx context.Context, remote string) ([]string, error) {
66+
return Reports(ctx, remote, "task")
67+
}
68+
69+
func Status(ctx context.Context, remote string, initialStatus common.Status) (common.Status, error) {
70+
reports, err := Reports(ctx, remote, "status")
71+
if err != nil {
72+
return initialStatus, err
73+
}
74+
75+
for _, report := range reports {
76+
var statusReport StatusReport
77+
if err := json.Unmarshal([]byte(report), &statusReport); err != nil {
78+
return initialStatus, err
79+
}
80+
if statusReport.Code != "" {
81+
if statusReport.Code == "0" {
82+
initialStatus[common.StatusCodeSucceeded] += 1
83+
} else {
84+
initialStatus[common.StatusCodeFailed] += 1
85+
}
86+
}
87+
}
88+
return initialStatus, nil
89+
}
90+
5691
func Transfer(ctx context.Context, source, destination string) error {
5792
sourceFileSystem, err := fs.NewFs(ctx, source)
5893
if err != nil {

‎task/common/values.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ type Status map[StatusCode]int
2626
type StatusCode string
2727

2828
const (
29-
StatusCodeRunning StatusCode = "running"
29+
StatusCodeActive StatusCode = "running"
30+
StatusCodeSucceeded StatusCode = "succeeded"
31+
StatusCodeFailed StatusCode = "failed"
3032
)
3133

3234
type Size struct {

‎task/gcp/resources/resource_instance_group_manager.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,8 @@ func (i *InstanceGroupManager) Read(ctx context.Context) error {
7575
}
7676

7777
i.Attributes.Addresses = []net.IP{}
78-
i.Attributes.Status = common.Status{common.StatusCodeRunning: 0}
78+
i.Attributes.Status = common.Status{common.StatusCodeActive: 0}
7979
for _, groupInstance := range groupInstances.Items {
80-
i.Attributes.Status[common.StatusCode(groupInstance.Status)]++
8180
if groupInstance.Status == "RUNNING" {
8281
instance, err := i.Client.Services.Compute.Instances.Get(i.Client.Credentials.ProjectID, i.Client.Region, filepath.Base(groupInstance.Instance)).Do()
8382
if err != nil {
@@ -86,7 +85,7 @@ func (i *InstanceGroupManager) Read(ctx context.Context) error {
8685
if address := net.ParseIP(instance.NetworkInterfaces[0].AccessConfigs[0].NatIP); address != nil {
8786
i.Attributes.Addresses = append(i.Attributes.Addresses, address)
8887
}
89-
i.Attributes.Status[common.StatusCodeRunning]++
88+
i.Attributes.Status[common.StatusCodeActive]++
9089
}
9190
}
9291

‎task/gcp/task.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,8 +352,12 @@ func (t *Task) Events(ctx context.Context) []common.Event {
352352
return t.Attributes.Events
353353
}
354354

355-
func (t *Task) Status(ctx context.Context) common.Status {
356-
return t.Attributes.Status
355+
func (t *Task) Status(ctx context.Context) (common.Status, error) {
356+
if err := t.Read(ctx); err != nil {
357+
return nil, err
358+
}
359+
360+
return machine.Status(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"], t.Attributes.Status)
357361
}
358362

359363
func (t *Task) GetKeyPair(ctx context.Context) (*ssh.DeterministicSSHKeyPair, error) {

0 commit comments

Comments
 (0)