Skip to content

Commit 74ffb4d

Browse files
chetanmehdubee
chetanmeh
authored and
dubee
committed
Treat action code as attachments (#3945)
* Treat action code as attachments for all newly created or update action runtimes * Store base64 encoded streams in raw form * Add test for exec deserialization compatibility * Support code stored in jar property * Use octet stream type for binary and text/plain otherwise Ignore contentType info from manifest * Reword exception message to state that code can be string on object * Drop rewrite of jar files as base64 encoded file * Remove unnecessary special handling of java code * Fix expected attachmentType * Add some more compat tests * Simplify json deserialization based on Markus patch * Use std charset * Add support for tweaking code size via CODE_SIZE If env CODE_SIZE is set then code would be padded with space * CODE_SIZE * Use attachment for php runtime also
1 parent 8327cd0 commit 74ffb4d

11 files changed

Lines changed: 550 additions & 158 deletions

File tree

‎ansible/files/runtimes.json‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@
88
"name": "nodejsaction",
99
"tag": "latest"
1010
},
11-
"deprecated": true
11+
"deprecated": true,
12+
"attached": {
13+
"attachmentName": "codefile",
14+
"attachmentType": "text/plain"
15+
}
1216
},
1317
{
1418
"kind": "nodejs:6",
@@ -19,6 +23,10 @@
1923
"tag": "latest"
2024
},
2125
"deprecated": false,
26+
"attached": {
27+
"attachmentName": "codefile",
28+
"attachmentType": "text/plain"
29+
},
2230
"stemCells": [{
2331
"count": 2,
2432
"memory": "256 MB"
@@ -32,7 +40,11 @@
3240
"name": "action-nodejs-v8",
3341
"tag": "latest"
3442
},
35-
"deprecated": false
43+
"deprecated": false,
44+
"attached": {
45+
"attachmentName": "codefile",
46+
"attachmentType": "text/plain"
47+
}
3648
}
3749
],
3850
"python": [
@@ -43,7 +55,11 @@
4355
"name": "python2action",
4456
"tag": "latest"
4557
},
46-
"deprecated": false
58+
"deprecated": false,
59+
"attached": {
60+
"attachmentName": "codefile",
61+
"attachmentType": "text/plain"
62+
}
4763
},
4864
{
4965
"kind": "python:2",
@@ -53,7 +69,11 @@
5369
"name": "python2action",
5470
"tag": "latest"
5571
},
56-
"deprecated": false
72+
"deprecated": false,
73+
"attached": {
74+
"attachmentName": "codefile",
75+
"attachmentType": "text/plain"
76+
}
5777
},
5878
{
5979
"kind": "python:3",
@@ -62,7 +82,11 @@
6282
"name": "python3action",
6383
"tag": "latest"
6484
},
65-
"deprecated": false
85+
"deprecated": false,
86+
"attached": {
87+
"attachmentName": "codefile",
88+
"attachmentType": "text/plain"
89+
}
6690
}
6791
],
6892
"swift": [
@@ -73,7 +97,11 @@
7397
"name": "swiftaction",
7498
"tag": "latest"
7599
},
76-
"deprecated": true
100+
"deprecated": true,
101+
"attached": {
102+
"attachmentName": "codefile",
103+
"attachmentType": "text/plain"
104+
}
77105
},
78106
{
79107
"kind": "swift:3",
@@ -82,7 +110,11 @@
82110
"name": "swift3action",
83111
"tag": "latest"
84112
},
85-
"deprecated": true
113+
"deprecated": true,
114+
"attached": {
115+
"attachmentName": "codefile",
116+
"attachmentType": "text/plain"
117+
}
86118
},
87119
{
88120
"kind": "swift:3.1.1",
@@ -91,7 +123,11 @@
91123
"name": "action-swift-v3.1.1",
92124
"tag": "latest"
93125
},
94-
"deprecated": false
126+
"deprecated": false,
127+
"attached": {
128+
"attachmentName": "codefile",
129+
"attachmentType": "text/plain"
130+
}
95131
},
96132
{
97133
"kind": "swift:4.1",
@@ -101,7 +137,11 @@
101137
"name": "action-swift-v4.1",
102138
"tag": "latest"
103139
},
104-
"deprecated": false
140+
"deprecated": false,
141+
"attached": {
142+
"attachmentName": "codefile",
143+
"attachmentType": "text/plain"
144+
}
105145
}
106146
],
107147
"java": [
@@ -115,8 +155,8 @@
115155
},
116156
"deprecated": false,
117157
"attached": {
118-
"attachmentName": "jarfile",
119-
"attachmentType": "application/java-archive"
158+
"attachmentName": "codefile",
159+
"attachmentType": "text/plain"
120160
},
121161
"requireMain": true
122162
}
@@ -130,6 +170,10 @@
130170
"prefix": "openwhisk",
131171
"name": "action-php-v7.1",
132172
"tag": "latest"
173+
},
174+
"attached": {
175+
"attachmentName": "codefile",
176+
"attachmentType": "text/plain"
133177
}
134178
},
135179
{
@@ -140,6 +184,10 @@
140184
"prefix": "openwhisk",
141185
"name": "action-php-v7.2",
142186
"tag": "latest"
187+
},
188+
"attached": {
189+
"attachmentName": "codefile",
190+
"attachmentType": "text/plain"
143191
}
144192
}
145193
],
@@ -148,6 +196,10 @@
148196
"kind": "ruby:2.5",
149197
"default": true,
150198
"deprecated": false,
199+
"attached": {
200+
"attachmentName": "codefile",
201+
"attachmentType": "text/plain"
202+
},
151203
"image": {
152204
"prefix": "openwhisk",
153205
"name": "action-ruby-v2.5",

‎common/scala/src/main/scala/whisk/core/entity/Exec.scala‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -140,14 +140,14 @@ protected[core] case class CodeExecMetaDataAsString(manifest: RuntimeManifest,
140140

141141
protected[core] case class CodeExecAsAttachment(manifest: RuntimeManifest,
142142
override val code: Attachment[String],
143-
override val entryPoint: Option[String])
143+
override val entryPoint: Option[String],
144+
override val binary: Boolean = false)
144145
extends CodeExec[Attachment[String]] {
145146
override val kind = manifest.kind
146147
override val image = manifest.image
147148
override val sentinelledLogs = manifest.sentinelledLogs.getOrElse(true)
148149
override val deprecated = manifest.deprecated.getOrElse(false)
149150
override val pull = false
150-
override lazy val binary = true
151151
override def codeAsJson = code.toJson
152152

153153
def inline(bytes: Array[Byte]): CodeExecAsAttachment = {
@@ -301,22 +301,24 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
301301
}
302302

303303
manifest.attached
304-
.map { a =>
305-
val jar: Attachment[String] = {
306-
// java actions once stored the attachment in "jar" instead of "code"
307-
obj.fields.get("code").orElse(obj.fields.get("jar"))
308-
} map {
309-
attFmt[String].read(_)
310-
} getOrElse {
304+
.map { _ =>
305+
// java actions once stored the attachment in "jar" instead of "code"
306+
val code = obj.fields.get("code").orElse(obj.fields.get("jar")).getOrElse {
311307
throw new DeserializationException(
312-
s"'code' must be a valid base64 string in 'exec' for '$kind' actions")
308+
s"'code' must be a string or attachment object defined in 'exec' for '$kind' actions")
313309
}
310+
val binary: Boolean = code match {
311+
case JsString(c) => isBinaryCode(c)
312+
case _ => obj.fields.get("binary").map(_.convertTo[Boolean]).getOrElse(false)
313+
}
314+
314315
val main = optMainField.orElse {
315316
if (manifest.requireMain.exists(identity)) {
316317
throw new DeserializationException(s"'main' must be a string defined in 'exec' for '$kind' actions")
317318
} else None
318319
}
319-
CodeExecAsAttachment(manifest, jar, main)
320+
321+
CodeExecAsAttachment(manifest, attFmt[String].read(code), main, binary)
320322
}
321323
.getOrElse {
322324
val code: String = obj.fields.get("code") match {

‎common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,11 @@ package whisk.core.entity
1919

2020
import java.io.ByteArrayInputStream
2121
import java.io.ByteArrayOutputStream
22+
import java.nio.charset.StandardCharsets
2223
import java.util.Base64
2324

25+
import akka.http.scaladsl.model.ContentTypes
26+
2427
import scala.concurrent.ExecutionContext
2528
import scala.concurrent.Future
2629
import scala.util.{Failure, Success, Try}
@@ -333,23 +336,27 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
333336
require(doc != null, "doc undefined")
334337
} map { _ =>
335338
doc.exec match {
336-
case exec @ CodeExecAsAttachment(_, Inline(code), _) =>
339+
case exec @ CodeExecAsAttachment(_, Inline(code), _, binary) =>
337340
implicit val logger = db.logging
338341
implicit val ec = db.executionContext
339342

340-
val stream = new ByteArrayInputStream(Base64.getDecoder().decode(code))
341-
val manifest = exec.manifest.attached.get
343+
val (bytes, attachmentType) = if (binary) {
344+
(Base64.getDecoder.decode(code), ContentTypes.`application/octet-stream`)
345+
} else {
346+
(code.getBytes(StandardCharsets.UTF_8), ContentTypes.`text/plain(UTF-8)`)
347+
}
348+
val stream = new ByteArrayInputStream(bytes)
342349
val oldAttachment = old
343350
.flatMap(_.exec match {
344-
case CodeExecAsAttachment(_, a: Attached, _) => Some(a)
345-
case _ => None
351+
case CodeExecAsAttachment(_, a: Attached, _, _) => Some(a)
352+
case _ => None
346353
})
347354

348355
super.putAndAttach(
349356
db,
350357
doc,
351358
(d, a) => d.copy(exec = exec.attach(a)).revision[WhiskAction](d.rev),
352-
manifest.attachmentType,
359+
attachmentType,
353360
stream,
354361
oldAttachment,
355362
Some { a: WhiskAction =>
@@ -378,12 +385,12 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
378385

379386
fa.flatMap { action =>
380387
action.exec match {
381-
case exec @ CodeExecAsAttachment(_, attached: Attached, _) =>
388+
case exec @ CodeExecAsAttachment(_, attached: Attached, _, binary) =>
382389
val boas = new ByteArrayOutputStream()
383-
val b64s = Base64.getEncoder().wrap(boas)
390+
val wrapped = if (binary) Base64.getEncoder().wrap(boas) else boas
384391

385-
getAttachment[A](db, action, attached, b64s, Some { a: WhiskAction =>
386-
b64s.close()
392+
getAttachment[A](db, action, attached, wrapped, Some { a: WhiskAction =>
393+
wrapped.close()
387394
val newAction = a.copy(exec = exec.inline(boas.toByteArray))
388395
newAction.revision(a.rev)
389396
newAction
@@ -397,7 +404,7 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
397404

398405
def attachmentHandler(action: WhiskAction, attached: Attached): WhiskAction = {
399406
val eu = action.exec match {
400-
case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _) =>
407+
case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _, _) =>
401408
require(
402409
attachmentName == attached.attachmentName,
403410
s"Attachment name '${attached.attachmentName}' does not match the expected name '$attachmentName'")

‎tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ class ColdBlockingInvokeSimulation extends Simulation {
3232
val host = sys.env("OPENWHISK_HOST")
3333

3434
val users: Int = sys.env("USERS").toInt
35+
val codeSize: Int = sys.env.getOrElse("CODE_SIZE", "0").toInt
3536
val seconds: FiniteDuration = sys.env.getOrElse("SECONDS", "10").toInt.seconds
3637
val actionsPerUser: Int = sys.env.getOrElse("ACTIONS_PER_USER", "5").toInt
3738

@@ -64,8 +65,7 @@ class ColdBlockingInvokeSimulation extends Simulation {
6465
openWhisk("Create action")
6566
.authenticate(uuid, key)
6667
.action(actionName)
67-
.create(FileUtils
68-
.readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8)))
68+
.create(actionCode))
6969
}.rendezVous(users)
7070
// Execute all actions for the given amount of time.
7171
.during(seconds) {
@@ -81,6 +81,13 @@ class ColdBlockingInvokeSimulation extends Simulation {
8181
}
8282
}
8383

84+
private def actionCode = {
85+
val code = FileUtils
86+
.readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8)
87+
//Pad the code with empty space to increase the stored code size
88+
if (codeSize > 0) code + " " * codeSize else code
89+
}
90+
8491
setUp(test.inject(atOnceUsers(users)))
8592
.protocols(openWhiskProtocol)
8693
// One failure will make the build yellow

0 commit comments

Comments
 (0)