Skip to content

Commit 021ff32

Browse files
davidheryantodavidheryanto
committed
Checkpoint
1 parent e14f89b commit 021ff32

File tree

16 files changed

+403
-157
lines changed

16 files changed

+403
-157
lines changed

‎core/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ Run the following maven command to start Feast Core GRPC service running on port
2020
```bash
2121
# Using configuration from src/main/resources/application.yml
2222
mvn spring-boot:run
23+
# Using configuration from custom location e.g. /tmp/config.application.yml
24+
mvn spring-boot:run -Dspring.config.location=/tmp/config.application.yml
2325
```
2426

2527
If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) installed, you can check that Feast Core is running

‎ingestion/src/main/java/feast/ingestion/util/StorageUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ public static void setupBigQuery(
162162

163163
// Return if there is an existing table
164164
Table table = bigquery.getTable(tableId);
165-
if (table == null || table.exists()) {
165+
if (table != null) {
166166
log.info(
167167
"Writing to existing BigQuery table '{}:{}.{}'",
168168
bigqueryProjectId,

‎protos/feast/serving/ServingService.proto‎

Lines changed: 12 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,14 @@ service ServingService {
3939

4040
// Get batch features asynchronously.
4141
//
42-
// The client should check and reload the status of the returned job
43-
// periodically to determine if the job has completed successfully or with
44-
// an error. If the job completes successfully, the client shoud call
45-
// GetBatchFeaturesFromCompletedJob to retrieve the feature values.
42+
// The client should check the status of the returned job periodically by
43+
// calling ReloadJobStatus to determine if the job has completed successfully
44+
// or with an error. If the job completes successfully i.e.
45+
// status = JOB_STATUS_DONE with no error, then the client can check
46+
// the file_uris for the location to download feature values data.
47+
// The client is assumed to have access to these file URIs.
4648
rpc GetBatchFeatures (GetFeaturesRequest) returns (GetBatchFeaturesResponse);
4749

48-
// Get the URI(s) to download batch feature values from a succesful download job.
49-
rpc GetBatchFeaturesFromCompletedJob (GetBatchFeaturesFromCompletedJobRequest) returns (GetBatchFeaturesFromCompletedJobResponse);
50-
5150
// Get the URI prefix where the client can upload files to be accessed by Feast serving.
5251
rpc GetStagingLocation (GetStagingLocationRequest) returns (GetStagingLocationResponse);
5352

@@ -165,23 +164,6 @@ message GetBatchFeaturesResponse {
165164
Job job = 1;
166165
}
167166

168-
message GetBatchFeaturesFromCompletedJobRequest {
169-
Job job = 1;
170-
}
171-
172-
message GetBatchFeaturesFromCompletedJobResponse {
173-
// The list of URIs for the files containing the batch feature values requested.
174-
//
175-
// Feast may retrieve the batch features and save them into multiple sharded
176-
// files for improving performance and reliability. The client is expected
177-
// to join these sharded files manually.
178-
repeated string download_uris = 1;
179-
180-
// Data format of the file. All files will have the same data format.
181-
// For CSV, the files will contain both feature values and a column header.
182-
DataFormat data_format = 2;
183-
}
184-
185167
message GetStagingLocationRequest {}
186168

187169
message GetStagingLocationResponse {
@@ -286,4 +268,10 @@ message Job {
286268
JobStatus status = 3;
287269
// Output only. If not empty, the job has failed with this error message.
288270
string error = 4;
271+
// Output only. The list of URIs for the files to be downloaded or
272+
// uploaded (depends on the job type) for this particular job.
273+
repeated string file_uris = 5;
274+
// Output only. The data format for all the files.
275+
// For CSV format, the files contain both feature values and a column header.
276+
DataFormat data_format = 6;
289277
}

‎serving/README.md‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,69 @@ If you have [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line
2121
grpc_cli ls localhost:6566
2222
grpc_cli call localhost:6566 GetFeastServingVersion ''
2323
grpc_cli call localhost:6566 GetFeastServingType ''
24+
```
25+
26+
```bash
27+
grpc_cli call localhost:6565 ApplyFeatureSet '
28+
feature_set {
29+
name: "driver"
30+
version: 1
31+
entities {
32+
name: "driver_id"
33+
value_type: STRING
34+
}
35+
features {
36+
name: "city"
37+
value_type: STRING
38+
}
39+
features {
40+
name: "booking_completed_count"
41+
value_type: INT64
42+
}
43+
source {
44+
type: KAFKA
45+
kafka_source_config {
46+
bootstrap_servers: "localhost:9092"
47+
}
48+
}
49+
}
50+
'
51+
52+
grpc_cli call localhost:6565 GetFeatureSets '
53+
filter {
54+
feature_set_name: "driver"
55+
feature_set_version: "1"
56+
}
57+
'
58+
59+
grpc_cli call localhost:6566 GetBatchFeatures '
60+
feature_sets {
61+
name: "driver4"
62+
version: 1
63+
feature_names: "booking_completed_count"
64+
max_age {
65+
seconds: 86400
66+
}
67+
}
68+
entity_dataset {
69+
entity_names: "driver_id"
70+
entity_dataset_rows {
71+
entity_timestamp {
72+
seconds: 1569873954
73+
}
74+
}
75+
}
76+
'
77+
```
78+
79+
```
80+
import pandas as pd
81+
import fastavro
82+
83+
with open("/tmp/000000000000.avro") as f:
84+
reader = fastavro.reader(f)
85+
records = [r for r in reader]
86+
df = pandas.DataFrame.from_records(records)
87+
df.shape
88+
df.head(5)
2489
```

‎serving/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,12 @@
232232
<version>1.91.0</version>
233233
</dependency>
234234

235+
<dependency>
236+
<groupId>com.google.cloud</groupId>
237+
<artifactId>google-cloud-storage</artifactId>
238+
<version>1.91.0</version>
239+
</dependency>
240+
235241
<!-- OpenCensus, needed by Google Cloud -->
236242
<dependency>
237243
<groupId>io.opencensus</groupId>

‎serving/src/main/java/feast/serving/FeastProperties.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import lombok.Getter;
99
import lombok.Setter;
1010
import org.springframework.boot.context.properties.ConfigurationProperties;
11-
import org.springframework.context.annotation.Configuration;
1211

1312
@Getter
1413
@Setter
@@ -23,4 +22,6 @@ public class FeastProperties {
2322
private boolean tracingEnabled;
2423
private String tracingTracerName;
2524
private String tracingServiceName;
25+
private String jobStagingLocation;
26+
private String jobStoreName;
2627
}

‎serving/src/main/java/feast/serving/configuration/InstrumentationConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public InstrumentationConfig(FeastProperties feastProperties) {
1717
}
1818

1919
@Bean
20-
public Tracer getTracer() {
20+
public Tracer tracer() {
2121
if (!feastProperties.isTracingEnabled()) {
2222
return NoopTracerFactory.create();
2323
}

‎serving/src/main/java/feast/serving/configuration/JobServiceConfig.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package feast.serving.configuration;
2+
3+
import feast.core.CoreServiceProto.GetStoresRequest;
4+
import feast.core.CoreServiceProto.GetStoresRequest.Filter;
5+
import feast.core.CoreServiceProto.GetStoresResponse;
6+
import feast.core.StoreProto.Store;
7+
import feast.core.StoreProto.Store.RedisConfig;
8+
import feast.core.StoreProto.Store.StoreType;
9+
import feast.serving.FeastProperties;
10+
import feast.serving.service.JobService;
11+
import feast.serving.service.RedisBackedJobService;
12+
import feast.serving.service.SpecService;
13+
import org.springframework.beans.factory.annotation.Autowired;
14+
import org.springframework.context.annotation.Bean;
15+
import org.springframework.context.annotation.Configuration;
16+
import redis.clients.jedis.JedisPool;
17+
18+
@Configuration
19+
public class JobServiceConfig {
20+
private FeastProperties feastProperties;
21+
22+
@Autowired
23+
public JobServiceConfig(FeastProperties feastProperties) {
24+
this.feastProperties = feastProperties;
25+
}
26+
27+
@Bean
28+
public JobService jobService(SpecService specService) {
29+
String jobStoreName = feastProperties.getJobStoreName();
30+
GetStoresResponse storesResponse =
31+
specService.getStores(
32+
GetStoresRequest.newBuilder()
33+
.setFilter(Filter.newBuilder().setName(jobStoreName).build())
34+
.build());
35+
36+
if (storesResponse.getStoreCount() < 1) {
37+
throw new IllegalArgumentException(
38+
String.format(
39+
"Cannot resolve Store from store name '%s'. Ensure the store name exists in Feast.",
40+
jobStoreName));
41+
}
42+
43+
assert storesResponse.getStoreCount() == 1;
44+
Store store = storesResponse.getStore(0);
45+
StoreType storeType = store.getType();
46+
JobService jobService = null;
47+
48+
switch (storeType) {
49+
case REDIS:
50+
RedisConfig redisConfig = store.getRedisConfig();
51+
JedisPool jedisPool = new JedisPool(redisConfig.getHost(), redisConfig.getPort());
52+
jobService = new RedisBackedJobService(jedisPool);
53+
case INVALID:
54+
case BIGQUERY:
55+
case CASSANDRA:
56+
case UNRECOGNIZED:
57+
throw new IllegalArgumentException(
58+
String.format(
59+
"Unsupported store type '%s' for job store name '%s'", storeType, jobStoreName));
60+
}
61+
62+
return jobService;
63+
}
64+
}

‎serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import com.google.cloud.bigquery.BigQuery;
44
import com.google.cloud.bigquery.BigQueryOptions;
5+
import com.google.cloud.storage.Storage;
6+
import com.google.cloud.storage.StorageOptions;
57
import feast.core.CoreServiceProto.GetStoresRequest;
68
import feast.core.CoreServiceProto.GetStoresRequest.Filter;
79
import feast.core.CoreServiceProto.GetStoresResponse;
@@ -11,6 +13,7 @@
1113
import feast.core.StoreProto.Store.StoreType;
1214
import feast.serving.FeastProperties;
1315
import feast.serving.service.BigQueryServingService;
16+
import feast.serving.service.JobService;
1417
import feast.serving.service.RedisServingService;
1518
import feast.serving.service.ServingService;
1619
import feast.serving.service.SpecService;
@@ -19,25 +22,30 @@
1922
import org.springframework.beans.factory.annotation.Autowired;
2023
import org.springframework.context.annotation.Bean;
2124
import org.springframework.context.annotation.Configuration;
22-
import org.springframework.context.annotation.DependsOn;
2325
import redis.clients.jedis.JedisPool;
2426
import redis.clients.jedis.JedisPoolConfig;
2527

2628
@Slf4j
2729
@Configuration
2830
public class ServingServiceConfig {
2931
private String feastStoreName;
32+
private String jobStagingLocation;
3033

3134
@Autowired
3235
public ServingServiceConfig(FeastProperties feastProperties) {
3336
feastStoreName = feastProperties.getStoreName();
37+
jobStagingLocation = feastProperties.getJobStagingLocation();
38+
if (!jobStagingLocation.contains("://")) {
39+
throw new IllegalArgumentException(
40+
String.format("jobStagingLocation is not a valid URI: %s", jobStagingLocation));
41+
}
3442
}
3543

3644
@Bean
37-
@DependsOn({"specService", "getTracer"})
3845
public ServingService servingService(
3946
FeastProperties feastProperties,
4047
SpecService specService,
48+
JobService jobService,
4149
Tracer tracer) {
4250
GetStoresResponse storesResponse =
4351
specService.getStores(
@@ -61,15 +69,24 @@ public ServingService servingService(
6169
JedisPoolConfig poolConfig = new JedisPoolConfig();
6270
poolConfig.setMaxTotal(feastProperties.getRedisPoolMaxSize());
6371
poolConfig.setMaxIdle(feastProperties.getRedisPoolMaxIdle());
64-
JedisPool jedisPool = new JedisPool(poolConfig, store.getRedisConfig().getHost(),
65-
store.getRedisConfig().getPort());
72+
JedisPool jedisPool =
73+
new JedisPool(
74+
poolConfig, store.getRedisConfig().getHost(), store.getRedisConfig().getPort());
6675
servingService = new RedisServingService(jedisPool, specService, tracer);
6776
break;
6877
case BIGQUERY:
6978
BigQueryConfig bqConfig = store.getBigqueryConfig();
7079
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
80+
Storage storage = StorageOptions.getDefaultInstance().getService();
7181
servingService =
72-
new BigQueryServingService(bigquery, bqConfig.getProjectId(), bqConfig.getDatasetId(), specService);
82+
new BigQueryServingService(
83+
bigquery,
84+
bqConfig.getProjectId(),
85+
bqConfig.getDatasetId(),
86+
specService,
87+
jobService,
88+
jobStagingLocation,
89+
storage);
7390
break;
7491
case CASSANDRA:
7592
case UNRECOGNIZED:

‎serving/src/main/java/feast/serving/configuration/SpecServiceConfig.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ public class SpecServiceConfig {
1919
private int feastCorePort;
2020
private static final int CACHE_REFRESH_RATE_MINUTES = 30;
2121

22-
2322
private ScheduledExecutorService scheduledExecutorService =
2423
Executors.newSingleThreadScheduledExecutor();
2524

@@ -32,8 +31,8 @@ public SpecServiceConfig(FeastProperties feastProperties) {
3231
@Bean
3332
public SpecService specService(FeastProperties feastProperties) {
3433
CoreSpecService coreService = new CoreSpecService(feastCoreHost, feastCorePort);
35-
CachedSpecService cachedSpecStorage = new CachedSpecService(coreService,
36-
feastProperties.getStoreName());
34+
CachedSpecService cachedSpecStorage =
35+
new CachedSpecService(coreService, feastProperties.getStoreName());
3736
// reload all specs including new ones periodically
3837
scheduledExecutorService.schedule(
3938
cachedSpecStorage::populateCache, CACHE_REFRESH_RATE_MINUTES, TimeUnit.MINUTES);

0 commit comments

Comments
 (0)