update to latest spec

Romeo Kienzer · Romeo Kienzer · commit c6edd60eb1b1 · 2021-09-17T12:39:54.000Z
diff --git a/component-library/transform/spark-csv-to-parquet.ipynb b/component-library/transform/spark-csv-to-parquet.ipynb
@@ -14,7 +14,15 @@
     "tags": []
    },
    "source": [
-    "# Converts a CSV file with header to parquet using ApacheSpark"
+    "# spark-csv-to-parquet"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "63d16770-79ac-4255-a2d0-29412418bebf",
+   "metadata": {},
+   "source": [
+    "Converts a CSV file with header to parquet using ApacheSpark"
    ]
   },
   {
@@ -46,28 +54,6 @@
     "fi"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "norman-memory",
-   "metadata": {
-    "papermill": {
-     "duration": 0.023625,
-     "end_time": "2021-03-22T20:29:25.776130",
-     "exception": false,
-     "start_time": "2021-03-22T20:29:25.752505",
-     "status": "completed"
-    },
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# @param data_dir temporal data storage for local execution\n",
-    "# @param data_csv csv path and file name (default: data.csv)\n",
-    "# @param data_parquet path and parquet file name (default: data.parquet)\n",
-    "# @param master url of master (default: local mode)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -107,9 +93,16 @@
    },
    "outputs": [],
    "source": [
+    "# source path and file name (default: data.csv)\n",
     "data_csv = os.environ.get('data_csv', 'data.csv')\n",
+    "\n",
+    "# destination path and parquet file name (default: data.parquet)\n",
     "data_parquet = os.environ.get('data_parquet', 'data.parquet')\n",
+    "\n",
+    "# url of master (default: local mode)\n",
     "master = os.environ.get('master', \"local[*]\")\n",
+    "\n",
+    "# temporal data storage for local execution\n",
     "data_dir = os.environ.get('data_dir', '../../data/')"
    ]
   },
@@ -122,19 +115,20 @@
    "source": [
     "# override parameters received from a potential call using %run magic\n",
     "parameters = list(\n",
-    "    map(\n",
-    "        lambda s: re.sub('$', '\"', s),\n",
-    "        map(\n",
-    "            lambda s: s.replace('=', '=\"'),\n",
-    "            filter(\n",
-    "                lambda s: s.find('=') > -1,\n",
-    "                sys.argv\n",
-    "            )\n",
-    "        )\n",
-    "    )\n",
+    "  map(\n",
+    "      lambda s: re.sub('$', '\"', s),\n",
+    "      map(\n",
+    "          lambda s: s.replace('=', '=\"'),\n",
+    "          filter(\n",
+    "              lambda s: s.find('=') > -1 and bool(re.match('[A-Za-z0-9_]*=[.\\/A-Za-z0-9]*', s)),\n",
+    "              sys.argv\n",
+    "          )\n",
+    "      )\n",
+    "  )\n",
     ")\n",
     "\n",
     "for parameter in parameters:\n",
+    "    logging.warning('Parameter: '+parameter) \n",
     "    exec(parameter)"
    ]
   },
