Skip to content
This repository was archived by the owner on Feb 24, 2023. It is now read-only.

Commit 1341d63

Browse files
cwbeckcwbeck
authored
Added support for AWS S3 as back-end storage (#14)
* Added support for AWS S3 as back-end storage * Removed comments / output
1 parent 6590b71 commit 1341d63

File tree

14 files changed

+242
-40
lines changed

14 files changed

+242
-40
lines changed

‎api/src/APIServer.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import BaseLogger from './backends/logging/abstractions/BaseLogger';
2222
import Scale8Setup from './bootstrap/Scale8Setup';
2323
import BaseConfig from './backends/configuration/abstractions/BaseConfig';
2424
import { Mode } from './enums/Mode';
25+
import BaseStorage from './backends/storage/abstractions/BaseStorage';
2526

2627
// noinspection JSUnusedLocalSymbols
2728
@injectable()
@@ -34,6 +35,7 @@ export default class APIServer {
3435
protected readonly gqlServer: ApolloServer;
3536
protected readonly routing: Routing;
3637
protected readonly config: BaseConfig;
38+
protected readonly storage: BaseStorage;
3739

3840
protected httpsServer?: https.Server;
3941
protected httpServer?: http.Server;
@@ -45,13 +47,15 @@ export default class APIServer {
4547
@inject(TYPES.Shell) shell: Shell,
4648
@inject(TYPES.Routing) routing: Routing,
4749
@inject(TYPES.BackendConfig) config: BaseConfig,
50+
@inject(TYPES.BackendStorage) storage: BaseStorage,
4851
) {
4952
this.resolverRegister = resolverRegister;
5053
this.typeDefRegister = typeDefRegister;
5154
this.logger = logger;
5255
this.shell = shell;
5356
this.routing = routing;
5457
this.config = config;
58+
this.storage = storage;
5559
this.gqlServer = this.getGQLServer();
5660
this.app = this.getApp();
5761
}
@@ -218,8 +222,7 @@ export default class APIServer {
218222
}
219223

220224
public async startServer(): Promise<void> {
221-
//this.logger.info(`Stripe Product & Plan Checks...`);
222-
//await container.resolve(StripeSetup).createProductsAndPlans();
225+
await this.storage.configure(); //we need to make sure our storage backend is properly configured
223226
this.logger.info(`Connecting to MongoDB...`).then();
224227
await this.shell.connect();
225228
await container.resolve(Scale8Setup).setup();

‎api/src/aws/S3Service.ts‎

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { injectable } from 'inversify';
22
import { S3 } from 'aws-sdk';
33
import { AwsRegion } from '../enums/AwsRegion';
4+
import { GetObjectOutput } from 'aws-sdk/clients/s3';
45

56
@injectable()
67
export default class S3Service {
@@ -10,31 +11,89 @@ export default class S3Service {
1011
accessKeyId: accessKeyId,
1112
secretAccessKey: secretAccessKey,
1213
},
13-
region: region.toLowerCase().replace(/_/g, '-'),
14+
region: this.getRegionFromAwsRegion(region),
1415
});
1516
}
1617

18+
public getRegionFromAwsRegion(region: AwsRegion): string {
19+
return region.toLowerCase().replace(/_/g, '-');
20+
}
21+
22+
public getSafeBucketName(bucketName: string) {
23+
return bucketName.replace(/[^a-z0-9]+/g, '-');
24+
}
25+
1726
public async bucketExists(s3Client: S3, bucketName: string): Promise<boolean> {
1827
return new Promise((resolve) => {
1928
s3Client.headBucket(
2029
{
21-
Bucket: bucketName,
30+
Bucket: this.getSafeBucketName(bucketName),
2231
},
2332
(err) => resolve(err === null),
2433
);
2534
});
2635
}
2736

37+
public async createBucket(
38+
s3Client: S3,
39+
bucketName: string,
40+
region: AwsRegion,
41+
): Promise<boolean> {
42+
const params = {
43+
Bucket: this.getSafeBucketName(bucketName),
44+
CreateBucketConfiguration: {
45+
LocationConstraint: this.getRegionFromAwsRegion(region),
46+
},
47+
};
48+
return new Promise((resolve, reject) => {
49+
s3Client.createBucket(params, (err) => {
50+
console.log(JSON.stringify(params));
51+
err === null ? resolve(true) : reject(err);
52+
});
53+
});
54+
}
55+
2856
public async isWriteable(s3Client: S3, bucketName: string): Promise<boolean> {
2957
return new Promise((resolve) => {
3058
s3Client.upload(
3159
{
32-
Bucket: bucketName,
60+
Bucket: this.getSafeBucketName(bucketName),
3361
Key: '/credential-check.txt',
3462
Body: 's8',
3563
},
3664
(err) => resolve(err === null),
3765
);
3866
});
3967
}
68+
69+
public async get(s3Client: S3, bucketName: string, key: string): Promise<GetObjectOutput> {
70+
return new Promise((resolve, reject) => {
71+
s3Client.getObject(
72+
{
73+
Bucket: this.getSafeBucketName(bucketName),
74+
Key: key,
75+
},
76+
(err, data) => {
77+
if (err === null) {
78+
resolve(data);
79+
} else {
80+
reject(err);
81+
}
82+
},
83+
);
84+
});
85+
}
86+
87+
public async put(s3Client: S3, params: S3.Types.PutObjectRequest): Promise<any> {
88+
params.Bucket = this.getSafeBucketName(params.Bucket);
89+
return new Promise((resolve, reject) => {
90+
s3Client.upload(params, (err: any) => {
91+
if (err === null) {
92+
resolve(true);
93+
} else {
94+
reject(err);
95+
}
96+
});
97+
});
98+
}
4099
}

‎api/src/backends/configuration/abstractions/BaseConfig.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import dotenv from 'dotenv';
33
import GenericError from '../../../errors/GenericError';
44
import { LogPriority } from '../../../enums/LogPriority';
55
import { Mode } from '../../../enums/Mode';
6+
import { AwsRegion } from '../../../enums/AwsRegion';
67

78
dotenv.config();
89

@@ -131,14 +132,14 @@ export default abstract class BaseConfig {
131132
return await this.getConfigEntryThrows('GC_PROJECT_ID');
132133
}
133134

134-
public async getGCAssetBucket(): Promise<string> {
135+
public async getAssetBucket(): Promise<string> {
135136
return await this.getConfigEntryOrElse(
136137
'ASSET_BUCKET',
137138
`scale8_com_${this.getEnvironment().toLowerCase()}_assets`,
138139
);
139140
}
140141

141-
public async getGCConfigsBucket(): Promise<string> {
142+
public async getConfigsBucket(): Promise<string> {
142143
return await this.getConfigEntryOrElse(
143144
'CONFIGS_BUCKET',
144145
`scale8_com_${this.getEnvironment().toLowerCase()}_configs`,
@@ -197,6 +198,10 @@ export default abstract class BaseConfig {
197198
return await this.getConfigEntryThrows('AWS_SECRET');
198199
}
199200

201+
public async getAwsRegion(): Promise<AwsRegion> {
202+
return (await this.getConfigEntryThrows('AWS_REGION')) as AwsRegion;
203+
}
204+
200205
public async getDatabaseUrl(): Promise<string> {
201206
return await this.getConfigEntryOrElse('MONGO_CONNECT_STRING', 'mongodb://127.0.0.1:27017');
202207
}

‎api/src/backends/storage/AmazonS3Storage.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { inject, injectable } from 'inversify';
2+
import BaseStorage, { StorageOptions } from './abstractions/BaseStorage';
3+
import TYPES from '../../container/IOC.types';
4+
import BaseConfig from '../configuration/abstractions/BaseConfig';
5+
import S3Service from '../../aws/S3Service';
6+
import { S3 } from 'aws-sdk';
7+
import GenericError from '../../errors/GenericError';
8+
import { LogPriority } from '../../enums/LogPriority';
9+
10+
@injectable()
11+
export default class AmazonS3Storage extends BaseStorage {
12+
@inject(TYPES.BackendConfig) private readonly config!: BaseConfig;
13+
@inject(TYPES.S3Service) private readonly s3Service!: S3Service;
14+
private storage: S3 | undefined;
15+
16+
protected async getStorage() {
17+
if (this.storage === undefined) {
18+
this.storage = this.s3Service.getS3Client(
19+
await this.config.getAwsId(),
20+
await this.config.getAwsSecret(),
21+
await this.config.getAwsRegion(),
22+
);
23+
}
24+
return this.storage;
25+
}
26+
27+
public async configure(): Promise<void> {
28+
await this.createBucketIfNotExists(await this.config.getAssetBucket());
29+
await this.createBucketIfNotExists(await this.config.getConfigsBucket());
30+
}
31+
32+
public async bucketExists(bucketName: string): Promise<boolean> {
33+
return this.s3Service.bucketExists(await this.getStorage(), bucketName);
34+
}
35+
36+
public async createBucket(bucketName: string): Promise<void> {
37+
const created = await this.s3Service.createBucket(
38+
await this.getStorage(),
39+
bucketName,
40+
await this.config.getAwsRegion(),
41+
);
42+
if (!created) {
43+
throw new GenericError(
44+
`Failed to create S3 bucket ${bucketName}`,
45+
LogPriority.ERROR,
46+
true,
47+
);
48+
}
49+
}
50+
51+
public async getAsString(bucketName: string, key: string): Promise<string> {
52+
const blob = await this.s3Service.get(await this.getStorage(), bucketName, key);
53+
return blob.Body === undefined ? '' : blob.Body.toString('utf-8');
54+
}
55+
56+
public async setAsString(
57+
bucketName: string,
58+
key: string,
59+
content: string,
60+
options?: StorageOptions,
61+
): Promise<void> {
62+
await this.s3Service.put(await this.getStorage(), {
63+
Bucket: bucketName,
64+
Key: key,
65+
Body: content,
66+
ContentType: options?.contentType || this.DEFAULT_MIME_TYPE,
67+
});
68+
}
69+
}

‎api/src/backends/storage/GoogleCloudStorage.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ export default class GoogleCloudStorage extends BaseStorage {
2020
}
2121

2222
public async configure(): Promise<void> {
23-
await this.createBucketIfNotExists(await this.config.getGCAssetBucket());
24-
await this.createBucketIfNotExists(await this.config.getGCConfigsBucket());
23+
await this.createBucketIfNotExists(await this.config.getAssetBucket());
24+
await this.createBucketIfNotExists(await this.config.getConfigsBucket());
2525
}
2626

2727
public async bucketExists(bucketName: string): Promise<boolean> {
@@ -39,23 +39,23 @@ export default class GoogleCloudStorage extends BaseStorage {
3939
});
4040
}
4141

42-
public async get(bucketName: string, key: string): Promise<any> {
42+
public async getAsString(bucketName: string, key: string): Promise<any> {
4343
const data = await (await this.getStorage()).bucket(bucketName).file(key).download();
4444
return data[0];
4545
}
4646

47-
public async put(
47+
public async setAsString(
4848
bucketName: string,
4949
key: string,
50-
blob: any,
50+
content: string,
5151
options?: StorageOptions,
5252
): Promise<void> {
5353
await (
5454
await this.getStorage()
5555
)
5656
.bucket(bucketName)
5757
.file(key)
58-
.save(blob, {
58+
.save(content, {
5959
contentType: options?.contentType || this.DEFAULT_MIME_TYPE,
6060
});
6161
}

‎api/src/backends/storage/MongoDBStorage.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ export default class MongoDBStorage extends BaseStorage {
2727
return this.shell.getCollection(`${bucketName}_bucket`);
2828
}
2929

30-
public async get(bucketName: string, key: string): Promise<any> {
30+
public async getAsString(bucketName: string, key: string): Promise<any> {
3131
const collection = await this.getCollection(bucketName);
3232
const doc = await collection.findOne({ key: key });
3333
if (doc) {
@@ -40,10 +40,10 @@ export default class MongoDBStorage extends BaseStorage {
4040
}
4141
}
4242

43-
public async put(
43+
public async setAsString(
4444
bucketName: string,
4545
key: string,
46-
content: string,
46+
config: string,
4747
options?: StorageOptions,
4848
): Promise<void> {
4949
this.logger.info(`Adding ${key} to ${bucketName}`).then();
@@ -53,7 +53,7 @@ export default class MongoDBStorage extends BaseStorage {
5353
{ key: key },
5454
{
5555
key: key,
56-
blob: content,
56+
blob: config,
5757
meta: { contentType: options?.contentType || this.DEFAULT_MIME_TYPE },
5858
},
5959
{ upsert: true },

‎api/src/backends/storage/abstractions/BaseStorage.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ export default abstract class BaseStorage {
2020
}
2121
}
2222

23-
public abstract put(
23+
public abstract setAsString(
2424
bucketName: string,
2525
key: string,
26-
blob: any,
26+
config: string,
2727
options?: StorageOptions,
2828
): Promise<void>;
2929

30-
public abstract get(bucketName: string, key: string): Promise<any>;
30+
public abstract getAsString(bucketName: string, key: string): Promise<string>;
3131
}

‎api/src/bootstrap/Scale8Setup.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,8 @@ export default class Scale8Setup {
131131
await Promise.all(
132132
Array.from(assets).map(async (value) => {
133133
const [fileName] = value;
134-
return this.backendStorage.put(
135-
await this.config.getGCAssetBucket(),
134+
return this.backendStorage.setAsString(
135+
await this.config.getAssetBucket(),
136136
`${revision.platformId.toString()}/${revision.id.toString()}-${fileName}`,
137137
fs.readFileSync(`${buildsPath}/${build}/${fileName}`, 'utf8'),
138138
{

‎api/src/utils/CertificateUtils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ export const uploadCertificate = async (
2121
})
2222
) {
2323
const upload = async (uri: string, data: string) =>
24-
storage.put(await config.getGCConfigsBucket(), uri, data, {
24+
storage.setAsString(await config.getConfigsBucket(), uri, data, {
2525
contentType: 'text/plain',
2626
});
2727

‎api/src/utils/EnvironmentUtils.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,8 @@ export const buildConfig = async (environment: Environment): Promise<void> => {
250250
);
251251

252252
const uploadTo = async (fileName: string) =>
253-
storage.put(
254-
await config.getGCConfigsBucket(),
253+
storage.setAsString(
254+
await config.getConfigsBucket(),
255255
`tag-domain/${fileName}`,
256256
JSON.stringify(await buildRevisionConfig(revision, environment)),
257257
{

0 commit comments

Comments
 (0)