Skip to content
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
690be0e
register cloud task triggers
GarrettBurroughs Jul 15, 2024
dfc96b0
Create and register task queue emulator
GarrettBurroughs Jul 15, 2024
0bd36f5
Add basic features of task queue emulator
GarrettBurroughs Jul 19, 2024
00cb33d
Implement a queue to improve performance of tasks emulator
GarrettBurroughs Jul 22, 2024
6b35b80
Remove unused import
GarrettBurroughs Jul 22, 2024
29b4a36
Register task queues from functions emulator
GarrettBurroughs Jul 22, 2024
46bdc76
change task queue endpoint to accept IDs in body instead of query params
GarrettBurroughs Jul 23, 2024
3fc9f1c
set enviornment variable when task queue emulator is running
GarrettBurroughs Jul 24, 2024
acd6c78
Remove bug where tasks were getting dispatched too fast
GarrettBurroughs Jul 24, 2024
e9c66a5
Merge remote-tracking branch 'origin' into gburroughs-taskqueue-emulator
GarrettBurroughs Jul 24, 2024
cee1b3d
Rework task enqueueing API and support task deletion
GarrettBurroughs Jul 26, 2024
db34cbd
properly support task deletion
GarrettBurroughs Jul 26, 2024
1c6f599
Refactor Task Dispatching Algorithm
GarrettBurroughs Jul 26, 2024
8957ee8
Add task deduplication for all tasks that have ever been added
GarrettBurroughs Jul 26, 2024
84685f6
Refactor Task Queue
GarrettBurroughs Jul 30, 2024
ff45600
Add tests for task queue emulator
GarrettBurroughs Jul 31, 2024
af218d8
respond to PR review
GarrettBurroughs Aug 5, 2024
faae51e
update firebase-config.json
GarrettBurroughs Aug 5, 2024
2c88c8e
use node-fetch
GarrettBurroughs Aug 5, 2024
d9126f1
fix tests
GarrettBurroughs Aug 5, 2024
3203c1b
Make tasks emulator start up whenever functions emulator starts
GarrettBurroughs Aug 6, 2024
7fcdd90
fix error message
GarrettBurroughs Aug 6, 2024
967a9bc
Add statistics endpoint
GarrettBurroughs Aug 8, 2024
03297d5
Add in global task deduplication
GarrettBurroughs Aug 8, 2024
fd97f9b
Add headers to request
GarrettBurroughs Aug 9, 2024
8ed0482
Review Changes
GarrettBurroughs Aug 12, 2024
0fe03f3
Remove console logs
GarrettBurroughs Aug 12, 2024
b9a0a63
Update backoff calculations
GarrettBurroughs Aug 14, 2024
b5813ae
Merge branch 'master' into gburroughs-taskqueue-emulator
GarrettBurroughs Aug 14, 2024
b978de0
Merge branch 'master' into gburroughs-taskqueue-emulator
blidd-google Aug 27, 2024
7b1d6a3
Merge branch 'master' into gburroughs-taskqueue-emulator
joehan Aug 27, 2024
2c4d236
add changelog
blidd-google Aug 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions schema/firebase-config.json
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,18 @@
},
"type": "object"
},
"tasks": {
"additionalProperties": false,
"properties": {
"host": {
"type": "string"
},
"port": {
"type": "number"
}
},
"type": "object"
},
"ui": {
"additionalProperties": false,
"properties": {
Expand Down
10 changes: 10 additions & 0 deletions src/emulator/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export const DEFAULT_PORTS: { [s in Emulators]: number } = {
storage: 9199,
eventarc: 9299,
dataconnect: 9399,
tasks: 9499,
};

export const FIND_AVAILBLE_PORT_BY_DEFAULT: Record<Emulators, boolean> = {
Expand All @@ -30,6 +31,7 @@ export const FIND_AVAILBLE_PORT_BY_DEFAULT: Record<Emulators, boolean> = {
extensions: false,
eventarc: true,
dataconnect: true,
tasks: true,
};

export const EMULATOR_DESCRIPTION: Record<Emulators, string> = {
Expand All @@ -46,6 +48,7 @@ export const EMULATOR_DESCRIPTION: Record<Emulators, string> = {
extensions: "Extensions Emulator",
eventarc: "Eventarc Emulator",
dataconnect: "Data Connect Emulator",
tasks: "Cloud Tasks Emulator",
};

export const DEFAULT_HOST = "localhost";
Expand Down Expand Up @@ -87,6 +90,9 @@ export class Constants {
// Environment variable to discover the eventarc emulator.
static CLOUD_EVENTARC_EMULATOR_HOST = "CLOUD_EVENTARC_EMULATOR_HOST";

// Environment variable to discover the tasks emulator.
static CLOUD_TASKS_EMULATOR_HOST = "CLOUD_TASKS_EMULATOR_HOST";

// Environment variable to discover the Emulator HUB
static FIREBASE_EMULATOR_HUB = "FIREBASE_EMULATOR_HUB";
static FIREBASE_GA_SESSION = "FIREBASE_GA_SESSION";
Expand All @@ -95,7 +101,9 @@ export class Constants {
static SERVICE_REALTIME_DATABASE = "firebaseio.com";
static SERVICE_PUBSUB = "pubsub.googleapis.com";
static SERVICE_EVENTARC = "eventarc.googleapis.com";
static SERVICE_CLOUD_TASKS = "cloudtasks.googleapis.com";
static SERVICE_FIREALERTS = "firebasealerts.googleapis.com";

// Note: the service name below are here solely for logging purposes.
// There is not an emulator available for these.
static SERVICE_ANALYTICS = "app-measurement.com";
Expand Down Expand Up @@ -127,6 +135,8 @@ export class Constants {
return "test lab";
case this.SERVICE_EVENTARC:
return "eventarc";
case this.SERVICE_CLOUD_TASKS:
return "tasks";
default:
return service;
}
Expand Down
14 changes: 13 additions & 1 deletion src/emulator/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import { HostingEmulator } from "./hostingEmulator";
import { PubsubEmulator } from "./pubsubEmulator";
import { StorageEmulator } from "./storage";
import { readFirebaseJson } from "../dataconnect/fileUtils";
import { TasksEmulator } from "./tasksEmulator";

const START_LOGGING_EMULATOR = utils.envOverride(
"START_LOGGING_EMULATOR",
Expand Down Expand Up @@ -371,11 +372,13 @@ export async function startAll(
// If we already know we need Functions (and Eventarc), assign them now.
listenConfig[Emulators.FUNCTIONS] = getListenConfig(options, Emulators.FUNCTIONS);
listenConfig[Emulators.EVENTARC] = getListenConfig(options, Emulators.EVENTARC);
listenConfig[Emulators.TASKS] = getListenConfig(options, Emulators.TASKS);
}
for (const emulator of ALL_EMULATORS) {
if (
emulator === Emulators.FUNCTIONS ||
emulator === Emulators.EVENTARC ||
emulator === Emulators.TASKS ||
// Same port as Functions, no need for separate assignment
emulator === Emulators.EXTENSIONS ||
(emulator === Emulators.UI && !showUI)
Expand Down Expand Up @@ -527,12 +530,13 @@ export async function startAll(
}

if (emulatableBackends.length) {
if (!listenForEmulator.functions || !listenForEmulator.eventarc) {
if (!listenForEmulator.functions || !listenForEmulator.eventarc || !listenForEmulator.tasks) {
// We did not know that we need Functions and Eventarc earlier but now we do.
listenForEmulator = await resolveHostAndAssignPorts({
...listenForEmulator,
functions: listenForEmulator.functions ?? getListenConfig(options, Emulators.FUNCTIONS),
eventarc: listenForEmulator.eventarc ?? getListenConfig(options, Emulators.EVENTARC),
tasks: listenForEmulator.eventarc ?? getListenConfig(options, Emulators.TASKS),
});
hubLogger.log("DEBUG", "late-assigned ports for functions and eventarc emulators", {
user: listenForEmulator,
Expand Down Expand Up @@ -588,6 +592,14 @@ export async function startAll(
port: eventarcAddr.port,
});
await startEmulator(eventarcEmulator);

const tasksAddr = legacyGetFirstAddr(Emulators.TASKS);
const tasksEmulator = new TasksEmulator({
host: tasksAddr.host,
port: tasksAddr.port,
});

await startEmulator(tasksEmulator);
Comment on lines +596 to +602
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Q: does this mean we always run the task emulator whenever we run the functions emulator?

I think this is fine for now, but I think an ideal solution would run the task emulator only if a task queue function is defined.

I don't know whether something like that is possible, but if you have time would love for you to see if that's even feasible.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the task queue emulator is run whenever the functions emulator is run.

There is the code here:

if (!this.controller.isRunning()) {

which only starts the listening logic of the task queue emulator once an actual task has been enqueued to avoid doing unnecessary updates while the user isn't doing anything with task queues in their functions code. The express server is still spun up every time the functions emulator starts though.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I wonder if we should instead have Task Queue Emulators be explicitly started, just like the Pub/sub emulator.

@blidd-google thoughts?

}

if (listenForEmulator.firestore) {
Expand Down
3 changes: 3 additions & 0 deletions src/emulator/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ export function setEnvVarsForEmulators(
case Emulators.EVENTARC:
env[Constants.CLOUD_EVENTARC_EMULATOR_HOST] = `http://${host}`;
break;
case Emulators.TASKS:
env[Constants.CLOUD_TASKS_EMULATOR_HOST] = host;
break;
}
}
}
37 changes: 37 additions & 0 deletions src/emulator/functionsEmulator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -652,6 +652,15 @@ export class FunctionsEmulator implements EmulatorInstance {
definition.name,
definition.region,
);
if (definition.taskQueueTrigger) {
added = await this.addTaskQueueTrigger(
this.args.projectId,
definition.region,
definition.name,
url,
definition.taskQueueTrigger,
);
}
} else if (definition.eventTrigger) {
const service: string = getFunctionService(definition);
const key = this.getTriggerKey(definition);
Expand Down Expand Up @@ -1193,6 +1202,34 @@ export class FunctionsEmulator implements EmulatorInstance {
return true;
}

addTaskQueueTrigger(
projectId: string,
location: string,
entryPoint: string,
defaultUri: string,
taskQueueTrigger: backend.TaskQueueTrigger,
): Promise<boolean> {
logger.debug(`addTaskQueueTrigger`, JSON.stringify(taskQueueTrigger));
if (!EmulatorRegistry.isRunning(Emulators.TASKS)) {
logger.debug(`addTaskQueueTrigger`, "TQ not running");
return Promise.resolve(false);
}
const bundle = {
...taskQueueTrigger,
defaultUri: defaultUri,
};

return EmulatorRegistry.client(Emulators.TASKS)
.post(`/projects/${projectId}/locations/${location}/queues/${entryPoint}`, bundle)
.then(() => {
return true;
})
.catch((err) => {
this.logger.log("WARN", "Error adding Task Queue function: " + err);
return false;
});
}

getProjectId(): string {
return this.args.projectId;
}
Expand Down
18 changes: 17 additions & 1 deletion src/emulator/functionsEmulatorShared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ export interface ParsedTriggerDefinition {
availableMemoryMb?: backend.MemoryOptions;
httpsTrigger?: any;
eventTrigger?: EventTrigger;
taskQueueTrigger?: backend.TaskQueueTrigger;
schedule?: EventSchedule;
blockingTrigger?: BlockingTrigger;
labels?: { [key: string]: any };
Expand Down Expand Up @@ -241,8 +242,20 @@ export function emulatedFunctionsFromEndpoints(
options: endpoint.blockingTrigger.options || {},
};
} else if (backend.isTaskQueueTriggered(endpoint)) {
// Just expose TQ trigger as HTTPS. Useful for debugging.
def.httpsTrigger = {};
def.taskQueueTrigger = {
retryConfig: {
maxAttempts: endpoint.taskQueueTrigger.retryConfig?.maxAttempts,
maxRetrySeconds: endpoint.taskQueueTrigger.retryConfig?.maxRetrySeconds,
maxBackoffSeconds: endpoint.taskQueueTrigger.retryConfig?.maxBackoffSeconds,
maxDoublings: endpoint.taskQueueTrigger.retryConfig?.maxDoublings,
minBackoffSeconds: endpoint.taskQueueTrigger.retryConfig?.minBackoffSeconds,
},
rateLimits: {
maxConcurrentDispatches: endpoint.taskQueueTrigger.rateLimits?.maxConcurrentDispatches,
maxDispatchesPerSecond: endpoint.taskQueueTrigger.rateLimits?.maxDispatchesPerSecond,
},
};
} else {
// All other trigger types are not supported by the emulator
// We leave both eventTrigger and httpTrigger attributes empty
Expand Down Expand Up @@ -347,6 +360,9 @@ export function getFunctionService(def: ParsedTriggerDefinition): string {
if (def.httpsTrigger) {
return "https";
}
if (def.taskQueueTrigger) {
return Constants.SERVICE_CLOUD_TASKS;
}

return "unknown";
}
Expand Down
1 change: 1 addition & 0 deletions src/emulator/portUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ const EMULATOR_CAN_LISTEN_ON_PRIMARY_ONLY: Record<PortName, boolean> = {
functions: true,
logging: true,
storage: true,
tasks: true,

// Only one hostname possible in .server mode, can switch to middleware later.
hosting: true,
Expand Down
1 change: 1 addition & 0 deletions src/emulator/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ export class EmulatorRegistry {
storage: 3.5,
eventarc: 3.6,
dataconnect: 3.7,
tasks: 3.8,

// Hub shuts down once almost everything else is done
hub: 4,
Expand Down
Loading