Skip to content

Commit 1cebc2d

Browse files
mrataj-gdstandevgd
mrataj-gd
authored and
standevgd
committed
add and list warehouse S3 credentials, closes gooddata#486
In order to use COPY FROM S3 functionality, users need to store their S3 credentials in Vertica. This adds support for POST and GET resource to allow the users to add/list credentials.
1 parent 057fd84 commit 1cebc2d

17 files changed

Lines changed: 781 additions & 4 deletions

‎src/main/java/com/gooddata/warehouse/WarehouseS3Credentials.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* Copyright (C) 2007-2017, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.warehouse;
5+
6+
import com.fasterxml.jackson.annotation.JsonCreator;
7+
import com.fasterxml.jackson.annotation.JsonIgnore;
8+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
9+
import com.fasterxml.jackson.annotation.JsonInclude;
10+
import com.fasterxml.jackson.annotation.JsonProperty;
11+
import com.fasterxml.jackson.annotation.JsonTypeInfo;
12+
import com.fasterxml.jackson.annotation.JsonTypeName;
13+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
14+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
15+
import com.gooddata.util.GoodDataToStringBuilder;
16+
import com.gooddata.util.ISODateTimeDeserializer;
17+
import com.gooddata.util.ISODateTimeSerializer;
18+
import org.joda.time.DateTime;
19+
import org.springframework.web.util.UriTemplate;
20+
21+
import java.util.Map;
22+
23+
import static com.gooddata.util.Validate.notNullState;
24+
25+
/**
26+
* Single warehouse S3 credentials record (identified uniquely by warehouse ID, region and access key)
27+
*/
28+
@JsonTypeInfo(include = JsonTypeInfo.As.WRAPPER_OBJECT, use = JsonTypeInfo.Id.NAME)
29+
@JsonTypeName("s3Credentials")
30+
@JsonIgnoreProperties(ignoreUnknown = true)
31+
@JsonInclude(JsonInclude.Include.NON_NULL)
32+
public class WarehouseS3Credentials {
33+
public static final String URI = WarehouseS3CredentialsList.URI + "/{region}/{accessKey}";
34+
public static final UriTemplate TEMPLATE = new UriTemplate(URI);
35+
36+
private static final String SELF_LINK = "self";
37+
38+
private final String region;
39+
40+
private final String accessKey;
41+
42+
private final DateTime updated;
43+
44+
private final String updatedBy;
45+
46+
private final String secretKey;
47+
48+
private final Map<String, String> links;
49+
50+
/**
51+
* Used to add new S3 credentials
52+
*
53+
* @param region S3 region
54+
* @param accessKey S3 access key
55+
* @param secretKey S3 secret key
56+
*/
57+
public WarehouseS3Credentials(final String region,
58+
final String accessKey,
59+
final String secretKey) {
60+
this(region, accessKey, secretKey, null, null, null);
61+
}
62+
63+
/**
64+
* Used to list saved S3 credentials (not intended for the end user)
65+
*
66+
* @param region S3 region
67+
* @param accessKey S3 access key
68+
* @param updatedBy URI of the user who updated the credentials last
69+
* @param updated last modification date
70+
*/
71+
public WarehouseS3Credentials(final String region,
72+
final String accessKey,
73+
final String updatedBy,
74+
final DateTime updated) {
75+
this(region, accessKey, null, updatedBy, updated, null);
76+
}
77+
78+
@JsonCreator
79+
WarehouseS3Credentials(@JsonProperty("region") final String region,
80+
@JsonProperty("accessKey") final String accessKey,
81+
@JsonProperty("secretKey") final String secretKey,
82+
@JsonProperty("updatedBy") final String updatedBy,
83+
@JsonProperty("updated") @JsonDeserialize(using = ISODateTimeDeserializer.class) final DateTime updated,
84+
@JsonProperty("links") final Map<String, String> links) {
85+
this.region = region;
86+
this.accessKey = accessKey;
87+
this.secretKey = secretKey;
88+
this.updatedBy = updatedBy;
89+
this.updated = updated;
90+
this.links = links;
91+
}
92+
93+
@JsonSerialize(using = ISODateTimeSerializer.class)
94+
public DateTime getUpdated() {
95+
return updated;
96+
}
97+
98+
/**
99+
* @return URI of the user who updated the credentials last
100+
*/
101+
public String getUpdatedBy() {
102+
return updatedBy;
103+
}
104+
105+
/**
106+
* not returned when listing
107+
* @return S3 secret key
108+
*/
109+
public String getSecretKey() {
110+
return secretKey;
111+
}
112+
113+
public String getAccessKey() {
114+
return accessKey;
115+
}
116+
117+
public String getRegion() {
118+
return region;
119+
}
120+
121+
public Map<String, String> getLinks() {
122+
return links;
123+
}
124+
125+
public WarehouseS3Credentials withLinks(final Map<String, String> links) {
126+
return new WarehouseS3Credentials(region, accessKey, secretKey, updatedBy, updated, links);
127+
}
128+
129+
@JsonIgnore
130+
public String getUri() {
131+
return notNullState(links, "links").get(SELF_LINK);
132+
}
133+
134+
@Override
135+
public String toString() {
136+
return GoodDataToStringBuilder.defaultToString(this, "secretKey");
137+
}
138+
}

‎src/main/java/com/gooddata/warehouse/WarehouseS3CredentialsList.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright (C) 2007-2017, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.warehouse;
5+
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
7+
import com.fasterxml.jackson.annotation.JsonTypeInfo;
8+
import com.fasterxml.jackson.annotation.JsonTypeName;
9+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
10+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
11+
import com.gooddata.collections.PageableList;
12+
import org.springframework.web.util.UriTemplate;
13+
14+
import java.util.List;
15+
import java.util.Map;
16+
17+
/**
18+
* List of warehouse S3 credentials for a given warehouse
19+
*/
20+
@JsonTypeInfo(include = JsonTypeInfo.As.WRAPPER_OBJECT, use = JsonTypeInfo.Id.NAME)
21+
@JsonTypeName("s3CredentialsList")
22+
@JsonIgnoreProperties(ignoreUnknown = true)
23+
@JsonDeserialize(using = WarehouseS3CredentialsListDeserializer.class)
24+
@JsonSerialize(using = WarehouseS3CredentialsListSerializer.class)
25+
public class WarehouseS3CredentialsList extends PageableList<WarehouseS3Credentials> {
26+
27+
static final String ROOT_NODE = "s3CredentialsList";
28+
public static final String URI = Warehouse.URI + "/s3";
29+
public static final UriTemplate TEMPLATE = new UriTemplate(URI);
30+
31+
public WarehouseS3CredentialsList(final List<WarehouseS3Credentials> items) {
32+
this(items, null);
33+
}
34+
35+
public WarehouseS3CredentialsList(final List<WarehouseS3Credentials> items,
36+
final Map<String, String> links) {
37+
super(items, null, links);
38+
}
39+
}

‎src/main/java/com/gooddata/warehouse/WarehouseS3CredentialsListDeserializer.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Copyright (C) 2007-2017, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.warehouse;
5+
6+
import com.gooddata.collections.PageableListDeserializer;
7+
import com.gooddata.collections.Paging;
8+
9+
import java.util.List;
10+
import java.util.Map;
11+
12+
class WarehouseS3CredentialsListDeserializer extends PageableListDeserializer<WarehouseS3CredentialsList, WarehouseS3Credentials> {
13+
14+
WarehouseS3CredentialsListDeserializer() {
15+
super(WarehouseS3Credentials.class);
16+
}
17+
18+
@Override
19+
protected WarehouseS3CredentialsList createList(List<WarehouseS3Credentials> items, Paging paging, Map<String, String> links) {
20+
return new WarehouseS3CredentialsList(items, links);
21+
}
22+
}

‎src/main/java/com/gooddata/warehouse/WarehouseS3CredentialsListSerializer.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
/*
2+
* Copyright (C) 2007-2017, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.warehouse;
5+
6+
import com.gooddata.collections.PageableListSerializer;
7+
8+
class WarehouseS3CredentialsListSerializer extends PageableListSerializer {
9+
10+
WarehouseS3CredentialsListSerializer() {
11+
super(WarehouseS3CredentialsList.ROOT_NODE);
12+
}
13+
}

‎src/main/java/com/gooddata/warehouse/WarehouseS3CredentialsNotFoundException.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright (C) 2007-2017, GoodData(R) Corporation. All rights reserved.
3+
*/
4+
package com.gooddata.warehouse;
5+
6+
import com.gooddata.GoodDataException;
7+
import com.gooddata.GoodDataRestException;
8+
9+
/**
10+
* Thrown when specific S3 credentials, identified by warehouse ID, region and access key cannot be found
11+
*/
12+
public class WarehouseS3CredentialsNotFoundException extends GoodDataException {
13+
14+
private final String warehouseS3CredentialsUri;
15+
16+
public WarehouseS3CredentialsNotFoundException(String uri, GoodDataRestException cause) {
17+
super("Warehouse S3 credentials " + uri + " not found", cause);
18+
this.warehouseS3CredentialsUri = uri;
19+
}
20+
21+
public String getWarehouseS3CredentialsUri() {
22+
return this.warehouseS3CredentialsUri;
23+
}
24+
}

‎src/main/java/com/gooddata/warehouse/WarehouseService.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,4 +437,122 @@ public WarehouseSchema getWarehouseSchemaByUri(final String uri) {
437437
public WarehouseSchema getDefaultWarehouseSchema(final Warehouse warehouse) {
438438
return getWarehouseSchemaByName(warehouse, DEFAULT_SCHEMA_NAME);
439439
}
440+
441+
/**
442+
* List S3 credentials for the Warehouse. Returns empty list if no credentials are found.
443+
*
444+
* @param warehouse warehouse to get S3 credentials for
445+
* @return PageableList with all S3 credentials belonging to the Warehouse (not null)
446+
*/
447+
public PageableList<WarehouseS3Credentials> listWarehouseS3Credentials(final Warehouse warehouse) {
448+
notNull(warehouse, "warehouse");
449+
450+
try {
451+
final String uri = getWarehouseS3CredentialsListUri(warehouse).toString();
452+
final WarehouseS3CredentialsList result = restTemplate.getForObject(uri, WarehouseS3CredentialsList.class);
453+
if (result == null) {
454+
return new PageableList<>();
455+
}
456+
return result;
457+
} catch (GoodDataException | RestClientException e) {
458+
throw new GoodDataException("Unable to list Warehouse S3 credentials for warehouse with uri: "
459+
+ warehouse.getUri(), e);
460+
}
461+
}
462+
463+
/**
464+
* Get S3 credentials for the Warehouse based on {@code region} and {@code accessKey}.
465+
*
466+
* @param warehouse warehouse to get S3 credentials for
467+
* @return single S3 credentials record (not null)
468+
* @throws WarehouseS3CredentialsNotFoundException if no S3 credentials for the given parameters were found
469+
*/
470+
public WarehouseS3Credentials getWarehouseS3Credentials(final Warehouse warehouse,
471+
final String region,
472+
final String accessKey) {
473+
notNull(warehouse, "warehouse");
474+
notEmpty(region, "region");
475+
notEmpty(accessKey, "accessKey");
476+
477+
final String uri = getWarehouseS3CredentialsUri(warehouse, region, accessKey).toString();
478+
479+
try {
480+
return restTemplate.getForObject(uri, WarehouseS3Credentials.class);
481+
} catch (GoodDataRestException e) {
482+
if (HttpStatus.NOT_FOUND.value() == e.getStatusCode()) {
483+
throw new WarehouseS3CredentialsNotFoundException(uri, e);
484+
} else {
485+
throw e;
486+
}
487+
} catch (RestClientException e) {
488+
throw new GoodDataException("Unable to get Warehouse S3 credentials " + uri, e);
489+
}
490+
}
491+
492+
/**
493+
* add new S3 credentials to the Warehouse
494+
*
495+
* @param warehouse warehouse to add credentials to
496+
* @param s3Credentials the credentials to store
497+
* @return added credentials (not null)
498+
*/
499+
public FutureResult<WarehouseS3Credentials> addS3CredentialsToWarehouse(final Warehouse warehouse,
500+
final WarehouseS3Credentials s3Credentials) {
501+
notNull(warehouse, "warehouse");
502+
notEmpty(warehouse.getId(), "warehouse.id");
503+
notNull(s3Credentials, "s3Credentials");
504+
505+
final WarehouseTask task;
506+
try {
507+
task = restTemplate.postForObject(WarehouseS3CredentialsList.URI, s3Credentials, WarehouseTask.class,
508+
warehouse.getId());
509+
} catch (GoodDataException | RestClientException e) {
510+
throw new GoodDataException("Unable add S3 credentials to warehouse " + warehouse.getId(), e);
511+
}
512+
if (task == null) {
513+
throw new GoodDataException("Empty response when S3 credentials POSTed to API");
514+
}
515+
516+
return new PollResult<>(this,
517+
new AbstractPollHandler<WarehouseTask, WarehouseS3Credentials>
518+
(task.getPollUri(), WarehouseTask.class, WarehouseS3Credentials.class) {
519+
520+
@Override
521+
public boolean isFinished(ClientHttpResponse response) throws IOException {
522+
return HttpStatus.CREATED.equals(response.getStatusCode());
523+
}
524+
525+
@Override
526+
public void handlePollResult(WarehouseTask pollResult) {
527+
try {
528+
final WarehouseS3Credentials created = restTemplate
529+
.getForObject(pollResult.getWarehouseS3CredentialsUri(),
530+
WarehouseS3Credentials.class);
531+
setResult(created);
532+
} catch (GoodDataException | RestClientException e) {
533+
throw new GoodDataException(
534+
"S3 credentials added to warehouse, but can't get them back, uri: "
535+
+ pollResult.getWarehouseUserUri(), e);
536+
}
537+
}
538+
539+
@Override
540+
public void handlePollException(final GoodDataRestException e) {
541+
throw new GoodDataException("Unable to add S3 credentials to warehouse, uri: "
542+
+ warehouse.getUri(), e);
543+
}
544+
});
545+
}
546+
547+
private URI getWarehouseS3CredentialsListUri(final Warehouse warehouse) {
548+
notEmpty(warehouse.getId(), "warehouse.id");
549+
550+
return WarehouseS3CredentialsList.TEMPLATE.expand(warehouse.getId());
551+
}
552+
553+
private URI getWarehouseS3CredentialsUri(final Warehouse warehouse, final String region, final String accessKey) {
554+
notEmpty(warehouse.getId(), "warehouse.id");
555+
556+
return WarehouseS3Credentials.TEMPLATE.expand(warehouse.getId(), region, accessKey);
557+
}
440558
}

‎src/main/java/com/gooddata/warehouse/WarehouseTask.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ class WarehouseTask {
2424
private static final String POLL_LINK = "poll";
2525
private static final String WAREHOUSE_LINK = "instance";
2626
private static final String WAREHOUSE_USER_LINK = "user";
27+
private static final String WAREHOUSE_S3_CREDENTIALS_LINK = "s3Credentials";
2728

2829
private final Map<String,String> links;
2930

@@ -58,4 +59,8 @@ String getWarehouseUserLink() {
5859
String getWarehouseUserUri() {
5960
return links.get(WAREHOUSE_USER_LINK);
6061
}
62+
63+
String getWarehouseS3CredentialsUri() {
64+
return links.get(WAREHOUSE_S3_CREDENTIALS_LINK);
65+
}
6166
}

0 commit comments

Comments
 (0)