Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "code",
5 "execution_count": 1,
6 "id": "d35cbf81-9ec9-45f7-b034-fa296d429f54",
7 "metadata": {},
8 "outputs": [],
9 "source": [
10 "from helper.credentials import *\n",
11 "from helper.dataextractor import *\n",
12 "from pyspark.sql import SparkSession\n",
13 "from pyspark.sql.functions import *\n",
14 "import datetime"
15 ]
16 },
17 {
18 "cell_type": "code",
19 "execution_count": 2,
20 "id": "56454870-c431-4fe0-88dd-26fdbf3821e8",
21 "metadata": {},
22 "outputs": [],
23 "source": [
24 "from pyspark import SparkContext"
25 ]
26 },
27 {
28 "cell_type": "code",
29 "execution_count": 3,
30 "id": "4722d484-7148-4efe-8711-0bd44027ff94",
31 "metadata": {},
32 "outputs": [],
33 "source": [
34 "s= datetime.datetime.now()"
35 ]
36 },
37 {
38 "cell_type": "code",
39 "execution_count": 4,
40 "id": "58e514fe-b19f-485d-a368-98da34f4f021",
41 "metadata": {},
42 "outputs": [],
43 "source": [
44 "# from pyspark import SparkConf\n",
45 "# conf = SparkConf().set(\"spark.hadoop.fs.s3a.access.key\", \"AKIATBUD4GB6PUTB2TOB\") \\\n",
46 "# .set(\"spark.hadoop.fs.s3a.secret.key\", \"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\") \\\n",
47 "# .set('packages', 'net.snowflake:snowflake-jdbc:3.13.26,net.snowflake:spark-snowflake_2.13:2.11.1-spark_3.3')"
48 ]
49 },
50 {
51 "cell_type": "code",
52 "execution_count": 5,
53 "id": "0762f644-5aad-4c86-8c4f-94ae30e8c92b",
54 "metadata": {},
55 "outputs": [],
56 "source": [
57 "#spark.stop()\n",
58 "common ='/driver/'\n",
59 "path1='snowflake-jdbc-3.13.26.jar'\n",
60 "path2='spark-snowflake_2.13-2.11.1-spark_3.3.jar'\n",
61 "p=path1+','+path2\n",
62 "spark = SparkSession.builder \\\n",
63 " .config('spark.jars', p) \\\n",
64 " .appName('dev') \\\n",
65 " .getOrCreate()"
66 ]
67 },
68 {
69 "cell_type": "code",
70 "execution_count": 6,
71 "id": "b9337f3d-3553-4140-9171-93cef9ad191b",
72 "metadata": {},
73 "outputs": [],
74 "source": [
75 "# #spark.stop()\n",
76 "# spark = SparkSession.builder \\\n",
77 "# .config('packages', 'net.snowflake:snowflake-jdbc:3.13.26,net.snowflake:spark-snowflake_2.13:2.11.1-spark_3.3') \\\n",
78 "# .config(\"spark.hadoop.fs.s3a.access.key\", \"AKIATBUD4GB6PUTB2TOB\")\\\n",
79 "# .config(\"spark.hadoop.fs.s3a.secret.key\", \"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\")\\\n",
80 "# .appName('dev') \\\n",
81 "# .getOrCreate()"
82 ]
83 },
84 {
85 "cell_type": "code",
86 "execution_count": 7,
87 "id": "b7dff7ae-d957-4183-9808-1618f4cc4203",
88 "metadata": {},
89 "outputs": [],
90 "source": [
91 "cred_manager= CredentialManager('secrets')\n",
92 "dataextractor= DataExtractor(spark, cred_manager)"
93 ]
94 },
95 {
96 "cell_type": "code",
97 "execution_count": 8,
98 "id": "951b1951-a38e-48c5-9a11-b95b8015ba75",
99 "metadata": {},
100 "outputs": [],
101 "source": [
102 "options={'table':'CUSTOMER_CATALOG',\n",
103 " 'warehouse':'COMPUTE', 'schema':'BUNGEE', 'database':'PETCO_BUNGEE_SHARE_INT'\n",
104 " }"
105 ]
106 },
107 {
108 "cell_type": "code",
109 "execution_count": null,
110 "id": "bda7611d-bdc4-48f4-9c50-8b58714ff9e7",
111 "metadata": {},
112 "outputs": [],
113 "source": []
114 },
115 {
116 "cell_type": "code",
117 "execution_count": 9,
118 "id": "6efa9696-85a1-4680-b69f-57f7a0fee010",
119 "metadata": {},
120 "outputs": [
121 {
122 "ename": "Py4JJavaError",
123 "evalue": "An error occurred while calling o54.load.\n: java.lang.ClassNotFoundException: Failed to find data source: net.snowflake.spark.snowflake. Please find packages at http://spark.apache.org/third-party-projects.html\n\tat org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:689)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSourceV2(DataSource.scala:743)\n\tat org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:266)\n\tat org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:226)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: java.lang.ClassNotFoundException: net.snowflake.spark.snowflake.DefaultSource\n\tat java.net.URLClassLoader.findClass(URLClassLoader.java:387)\n\tat java.lang.ClassLoader.loadClass(ClassLoader.java:418)\n\tat java.lang.ClassLoader.loadClass(ClassLoader.java:351)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$lookupDataSource$5(DataSource.scala:663)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$lookupDataSource$4(DataSource.scala:663)\n\tat scala.util.Failure.orElse(Try.scala:224)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:663)\n\t... 14 more\n",
124 "output_type": "error",
125 "traceback": [
126 "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
127 "\u001b[0;31mPy4JJavaError\u001b[0m Traceback (most recent call last)",
128 "\u001b[0;32m<ipython-input-9-ef01808aaf97>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mdataextractor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'petco-snowflake'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0;34m'cast(product_id as double)%2 = 0'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
129 "\u001b[0;32m~/workspace/jupyter_workspace/dev/helper/dataextractor.py\u001b[0m in \u001b[0;36mextract_data\u001b[0;34m(self, source, options, pushdown_predicate, columns)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspark_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'net.snowflake.spark.snowflake'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0msfOptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0;34m.\u001b[0m\u001b[0moption\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"query\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0;34m'aurora'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
130 "\u001b[0;32m~/spark/python/pyspark/sql/readwriter.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self, path, format, schema, **options)\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPythonUtils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoSeq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,\n",
131 "\u001b[0;32m~/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1303\u001b[0m \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1304\u001b[0m return_value = get_return_value(\n\u001b[0;32m-> 1305\u001b[0;31m answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m 1306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1307\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
132 "\u001b[0;32m~/spark/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 112\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0mconverted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
133 "\u001b[0;32m~/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m 326\u001b[0m raise Py4JJavaError(\n\u001b[1;32m 327\u001b[0m \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m 329\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m raise Py4JError(\n",
134 "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o54.load.\n: java.lang.ClassNotFoundException: Failed to find data source: net.snowflake.spark.snowflake. Please find packages at http://spark.apache.org/third-party-projects.html\n\tat org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:689)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSourceV2(DataSource.scala:743)\n\tat org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:266)\n\tat org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:226)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: java.lang.ClassNotFoundException: net.snowflake.spark.snowflake.DefaultSource\n\tat java.net.URLClassLoader.findClass(URLClassLoader.java:387)\n\tat java.lang.ClassLoader.loadClass(ClassLoader.java:418)\n\tat java.lang.ClassLoader.loadClass(ClassLoader.java:351)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$lookupDataSource$5(DataSource.scala:663)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$lookupDataSource$4(DataSource.scala:663)\n\tat scala.util.Failure.orElse(Try.scala:224)\n\tat org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:663)\n\t... 14 more\n"
135 ]
136 }
137 ],
138 "source": [
139 "df= dataextractor.extract_data('petco-snowflake', options , 'cast(product_id as double)%2 = 0')"
140 ]
141 },
142 {
143 "cell_type": "code",
144 "execution_count": 10,
145 "id": "cd909b6c-a1d1-4c0f-8a69-428d0deaa100",
146 "metadata": {},
147 "outputs": [
148 {
149 "name": "stdout",
150 "output_type": "stream",
151 "text": [
152 "root\n",
153 " |-- DATETIME: timestamp (nullable = false)\n",
154 " |-- PRODUCT_ID: decimal(38,0) (nullable = false)\n",
155 " |-- PARENT_PRODUCT_PART_DOUBLE: string (nullable = true)\n",
156 " |-- UPC: string (nullable = true)\n",
157 " |-- PRODUCT_NAME: string (nullable = true)\n",
158 " |-- PRODUCT_DESCRIPTION: string (nullable = true)\n",
159 " |-- PRODUCT_SIZE: string (nullable = true)\n",
160 " |-- PRODUCT_COLOR: string (nullable = true)\n",
161 " |-- PRODUCT_PURCHASE_BRAND: string (nullable = true)\n",
162 " |-- PRIMARY_VENDOR_NAME: string (nullable = true)\n",
163 " |-- PRODUCT_CATEGORY_LEVEL1: string (nullable = true)\n",
164 " |-- PRODUCT_CATEGORY_LEVEL2: string (nullable = true)\n",
165 " |-- PRODUCT_CATEGORY_LEVEL3: string (nullable = true)\n",
166 " |-- PRODUCT_CATEGORY_LEVEL4: string (nullable = true)\n",
167 " |-- PRODUCT_CATEGORY_LEVEL5: string (nullable = true)\n",
168 " |-- PRODUCT_CATEGORY_LEVEL6: string (nullable = true)\n",
169 " |-- PRODUCT_FULLIMAGE: string (nullable = true)\n",
170 " |-- PRODUCT_URL: string (nullable = true)\n",
171 " |-- PRODUCT_DISCONTINUED_FLAG: date (nullable = true)\n",
172 " |-- PRODUCT_PUBLISHED_FLAG: decimal(38,0) (nullable = true)\n",
173 " |-- PRODUCT_WEIGHT: string (nullable = true)\n",
174 " |-- PRODUCT_WEIGHT_UOM: string (nullable = true)\n",
175 " |-- PRODUCT_LENGTH: decimal(10,2) (nullable = true)\n",
176 " |-- PRODUCT_WIDTH: decimal(10,2) (nullable = true)\n",
177 " |-- PRODUCT_HEIGHT: decimal(10,2) (nullable = true)\n",
178 "\n"
179 ]
180 }
181 ],
182 "source": [
183 "df.printSchema()"
184 ]
185 },
186 {
187 "cell_type": "code",
188 "execution_count": null,
189 "id": "a3dcd354-fb7d-4806-bd34-5e3e3d2ef0c3",
190 "metadata": {},
191 "outputs": [],
192 "source": []
193 },
194 {
195 "cell_type": "code",
196 "execution_count": 11,
197 "id": "cb044fbf-5e1a-4358-a305-f52c29b0ce2d",
198 "metadata": {},
199 "outputs": [],
200 "source": [
201 "grouped= df.groupby('PRODUCT_COLOR').agg(count('PRODUCT_ID').alias('count') )"
202 ]
203 },
204 {
205 "cell_type": "code",
206 "execution_count": 12,
207 "id": "ce22da84-af22-4320-addf-2c898fe887b1",
208 "metadata": {},
209 "outputs": [
210 {
211 "name": "stdout",
212 "output_type": "stream",
213 "text": [
214 "+----------------+------+\n",
215 "| PRODUCT_COLOR| count|\n",
216 "+----------------+------+\n",
217 "| null|109251|\n",
218 "|Other or Unknown| 6179|\n",
219 "| Multi| 6052|\n",
220 "| Blue| 3602|\n",
221 "| Pink| 2816|\n",
222 "| Red| 2634|\n",
223 "| Black| 2599|\n",
224 "| Gray| 1967|\n",
225 "| Brown| 1742|\n",
226 "| Green| 1662|\n",
227 "| White or Ivory| 1071|\n",
228 "| Orange| 601|\n",
229 "| Purple| 571|\n",
230 "| Yellow| 475|\n",
231 "| Metallic| 338|\n",
232 "| Transparent| 67|\n",
233 "+----------------+------+\n",
234 "\n"
235 ]
236 }
237 ],
238 "source": [
239 "grouped.repartition(1).orderBy(grouped['count'].desc()).show()"
240 ]
241 },
242 {
243 "cell_type": "code",
244 "execution_count": 13,
245 "id": "9972dabe-84d8-49e0-9693-a2f5d35b9f8e",
246 "metadata": {},
247 "outputs": [
248 {
249 "name": "stdout",
250 "output_type": "stream",
251 "text": [
252 "+----------------------+\n",
253 "|PRODUCT_PUBLISHED_FLAG|\n",
254 "+----------------------+\n",
255 "| 1|\n",
256 "| null|\n",
257 "| 0|\n",
258 "+----------------------+\n",
259 "\n"
260 ]
261 }
262 ],
263 "source": [
264 "df.select('PRODUCT_PUBLISHED_FLAG').distinct().show()"
265 ]
266 },
267 {
268 "cell_type": "code",
269 "execution_count": 14,
270 "id": "60fcc060-70ee-4dc0-b112-7f5d0c65419d",
271 "metadata": {},
272 "outputs": [],
273 "source": [
274 "df =df.withColumn('is_discontinued' , when(isnull(col('PRODUCT_DISCONTINUED_FLAG')), True).otherwise(False) )"
275 ]
276 },
277 {
278 "cell_type": "code",
279 "execution_count": 15,
280 "id": "b66dce0c-8a2f-4e59-b329-54c86c238c80",
281 "metadata": {},
282 "outputs": [],
283 "source": [
284 "df =df.withColumn('is_active' , when(isnull(col('PRODUCT_DISCONTINUED_FLAG')), 0).otherwise(1) )"
285 ]
286 },
287 {
288 "cell_type": "code",
289 "execution_count": 16,
290 "id": "5396ff66-7b0f-4568-82b5-0c17f4a99e29",
291 "metadata": {},
292 "outputs": [
293 {
294 "name": "stdout",
295 "output_type": "stream",
296 "text": [
297 "+---------+---------------+-----------------+\n",
298 "|is_active|is_discontinued|count(PRODUCT_ID)|\n",
299 "+---------+---------------+-----------------+\n",
300 "| 0| true| 35090|\n",
301 "| 1| false| 106537|\n",
302 "+---------+---------------+-----------------+\n",
303 "\n"
304 ]
305 }
306 ],
307 "source": [
308 "df.groupby('is_active', 'is_discontinued').agg(count(col('PRODUCT_ID'))).show()"
309 ]
310 },
311 {
312 "cell_type": "code",
313 "execution_count": 17,
314 "id": "91e016f5-26ff-49ec-92cc-cf85a7a08d43",
315 "metadata": {},
316 "outputs": [],
317 "source": [
318 "grouped=df.filter(col('is_active')==1).groupby('PRODUCT_PURCHASE_BRAND').agg(count(col('PRODUCT_ID')).alias('count'))"
319 ]
320 },
321 {
322 "cell_type": "code",
323 "execution_count": 18,
324 "id": "898ea84a-cdff-457c-b46c-c68d725c0b8d",
325 "metadata": {},
326 "outputs": [
327 {
328 "name": "stdout",
329 "output_type": "stream",
330 "text": [
331 "+----------------------+-----+\n",
332 "|PRODUCT_PURCHASE_BRAND|count|\n",
333 "+----------------------+-----+\n",
334 "| UNASSIGNED|25060|\n",
335 "| PETCO BRAND| 6303|\n",
336 "| SIMPLY SHE (CUSTO...| 3313|\n",
337 "| DICKENS CLOSET| 2947|\n",
338 "| BOND & CO.| 1844|\n",
339 "+----------------------+-----+\n",
340 "only showing top 5 rows\n",
341 "\n"
342 ]
343 }
344 ],
345 "source": [
346 "grouped.repartition(1).orderBy(grouped['count'].desc()).show(5)"
347 ]
348 },
349 {
350 "cell_type": "code",
351 "execution_count": 19,
352 "id": "b8530fd2-7cef-4928-bde9-54d22135a87e",
353 "metadata": {},
354 "outputs": [
355 {
356 "data": {
357 "text/plain": [
358 "1830"
359 ]
360 },
361 "execution_count": 19,
362 "metadata": {},
363 "output_type": "execute_result"
364 }
365 ],
366 "source": [
367 "grouped.count()"
368 ]
369 },
370 {
371 "cell_type": "code",
372 "execution_count": 20,
373 "id": "3a5be4c2-9a8b-4c2d-986c-85705284294a",
374 "metadata": {},
375 "outputs": [],
376 "source": [
377 "grouped2=df.filter(col('is_active')==1).groupby('PRODUCT_PURCHASE_BRAND')\\\n",
378 " .agg(collect_set(substring(col('UPC'),1,5)).alias('upc5'),\n",
379 " count_distinct( substring(col('UPC'),1,5)).alias('count_upc5')\n",
380 " )"
381 ]
382 },
383 {
384 "cell_type": "code",
385 "execution_count": 21,
386 "id": "697a1c98-0d40-4436-af61-7de33dd90453",
387 "metadata": {},
388 "outputs": [
389 {
390 "data": {
391 "text/plain": [
392 "1830"
393 ]
394 },
395 "execution_count": 21,
396 "metadata": {},
397 "output_type": "execute_result"
398 }
399 ],
400 "source": [
401 "grouped2.count()"
402 ]
403 },
404 {
405 "cell_type": "code",
406 "execution_count": 22,
407 "id": "1e8c028c-ad64-475f-9c93-17a1889f0626",
408 "metadata": {},
409 "outputs": [],
410 "source": [
411 "final_df= grouped.repartition(1).join(grouped2.repartition(1),['PRODUCT_PURCHASE_BRAND'], 'inner')"
412 ]
413 },
414 {
415 "cell_type": "code",
416 "execution_count": 23,
417 "id": "83d9dc50-131d-4ead-a1df-bde717bd266c",
418 "metadata": {},
419 "outputs": [
420 {
421 "name": "stdout",
422 "output_type": "stream",
423 "text": [
424 "+----------------------+-----+--------------------+----------+\n",
425 "|PRODUCT_PURCHASE_BRAND|count| upc5|count_upc5|\n",
426 "+----------------------+-----+--------------------+----------+\n",
427 "| 2 FOR $5| 114|[80044, 74677, 61...| 4|\n",
428 "| 2 FOR 5| 2| [74677]| 1|\n",
429 "| 26 BARS AND A BAND| 57| [71812, 70510]| 2|\n",
430 "| 3 DOG PET SUPPLY| 1| [85975]| 1|\n",
431 "| 3M MASKING & PACK...| 39|[50051, 02120, 05...| 5|\n",
432 "+----------------------+-----+--------------------+----------+\n",
433 "only showing top 5 rows\n",
434 "\n"
435 ]
436 }
437 ],
438 "source": [
439 "final_df.show(5)"
440 ]
441 },
442 {
443 "cell_type": "code",
444 "execution_count": null,
445 "id": "9d4c489b-d2c8-4542-ad36-4a8410383908",
446 "metadata": {},
447 "outputs": [],
448 "source": []
449 },
450 {
451 "cell_type": "code",
452 "execution_count": 24,
453 "id": "779a43d6-b977-4152-861d-da280edbec26",
454 "metadata": {},
455 "outputs": [
456 {
457 "name": "stdout",
458 "output_type": "stream",
459 "text": [
460 "0:01:18.503431\n"
461 ]
462 }
463 ],
464 "source": [
465 "print(datetime.datetime.now() -s )"
466 ]
467 },
468 {
469 "cell_type": "code",
470 "execution_count": 25,
471 "id": "63a0c9ac-7728-4976-82db-912d369b818a",
472 "metadata": {},
473 "outputs": [],
474 "source": [
475 "s=datetime.datetime.now()"
476 ]
477 },
478 {
479 "cell_type": "code",
480 "execution_count": 31,
481 "id": "10555399-57e1-4de7-9841-16ab29604526",
482 "metadata": {},
483 "outputs": [],
484 "source": [
485 "sc = spark.sparkContext\n",
486 "access_key='AKIATBUD4GB6PUTB2TOB'\n",
487 "secret_key='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'\n",
488 "# remove this block if use core-site.xml and env variable\n",
489 "sc._jsc.hadoopConfiguration().set(\"fs.s3.awsAccessKeyId\", access_key)\n",
490 "sc._jsc.hadoopConfiguration().set(\"fs.s3n.awsAccessKeyId\", access_key)\n",
491 "sc._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\", access_key)\n",
492 "sc._jsc.hadoopConfiguration().set(\"fs.s3.awsSecretAccessKey\", secret_key)\n",
493 "sc._jsc.hadoopConfiguration().set(\"fs.s3n.awsSecretAccessKey\", secret_key)\n",
494 "sc._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\", secret_key)\n",
495 "sc._jsc.hadoopConfiguration().set(\"fs.s3n.impl\", \"org.apache.hadoop.fs.s3native.NativeS3FileSystem\")\n",
496 "sc._jsc.hadoopConfiguration().set(\"fs.s3a.impl\", \"org.apache.hadoop.fs.s3a.S3AFileSystem\")\n",
497 "sc._jsc.hadoopConfiguration().set(\"fs.s3.impl\", \"org.apache.hadoop.fs.s3.S3FileSystem\")"
498 ]
499 },
500 {
501 "cell_type": "code",
502 "execution_count": 34,
503 "id": "6b43c7de-38b7-4e92-b6fe-48ebe4a2c483",
504 "metadata": {},
505 "outputs": [
506 {
507 "ename": "Py4JJavaError",
508 "evalue": "An error occurred while calling o160.parquet.\n: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme \"s3fs\"\r\n\tat org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3443)\r\n\tat org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3466)\r\n\tat org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521)\r\n\tat org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540)\r\n\tat org.apache.hadoop.fs.Path.getFileSystem(Path.java:365)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWritingFileFormat(DataSource.scala:461)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWriting(DataSource.scala:558)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:390)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:363)\r\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)\r\n\tat org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:793)\r\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)\r\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:578)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\r\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\r\n\tat java.base/java.lang.Thread.run(Thread.java:1589)\r\n",
509 "output_type": "error",
510 "traceback": [
511 "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
512 "\u001b[1;31mPy4JJavaError\u001b[0m Traceback (most recent call last)",
513 "Cell \u001b[1;32mIn[34], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mfinal_df\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparquet\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43ms3fs://shubham-test-bucket/testing/data/petco\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
514 "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\pyspark\\sql\\readwriter.py:1140\u001b[0m, in \u001b[0;36mDataFrameWriter.parquet\u001b[1;34m(self, path, mode, partitionBy, compression)\u001b[0m\n\u001b[0;32m 1138\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpartitionBy(partitionBy)\n\u001b[0;32m 1139\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_opts(compression\u001b[38;5;241m=\u001b[39mcompression)\n\u001b[1;32m-> 1140\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jwrite\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparquet\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n",
515 "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\py4j\\java_gateway.py:1321\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m 1315\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1316\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1317\u001b[0m args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1318\u001b[0m proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[0;32m 1320\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client\u001b[38;5;241m.\u001b[39msend_command(command)\n\u001b[1;32m-> 1321\u001b[0m return_value \u001b[38;5;241m=\u001b[39m \u001b[43mget_return_value\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 1322\u001b[0m \u001b[43m \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtarget_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1324\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n\u001b[0;32m 1325\u001b[0m temp_arg\u001b[38;5;241m.\u001b[39m_detach()\n",
516 "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\pyspark\\sql\\utils.py:190\u001b[0m, in \u001b[0;36mcapture_sql_exception.<locals>.deco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m 188\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdeco\u001b[39m(\u001b[38;5;241m*\u001b[39ma: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m 189\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 190\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 191\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m Py4JJavaError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m 192\u001b[0m converted \u001b[38;5;241m=\u001b[39m convert_exception(e\u001b[38;5;241m.\u001b[39mjava_exception)\n",
517 "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\py4j\\protocol.py:326\u001b[0m, in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m 324\u001b[0m value \u001b[38;5;241m=\u001b[39m OUTPUT_CONVERTER[\u001b[38;5;28mtype\u001b[39m](answer[\u001b[38;5;241m2\u001b[39m:], gateway_client)\n\u001b[0;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m answer[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m==\u001b[39m REFERENCE_TYPE:\n\u001b[1;32m--> 326\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Py4JJavaError(\n\u001b[0;32m 327\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[0;32m 328\u001b[0m \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name), value)\n\u001b[0;32m 329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Py4JError(\n\u001b[0;32m 331\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m. Trace:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{3}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[0;32m 332\u001b[0m \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name, value))\n",
518 "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling o160.parquet.\n: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme \"s3fs\"\r\n\tat org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3443)\r\n\tat org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3466)\r\n\tat org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521)\r\n\tat org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540)\r\n\tat org.apache.hadoop.fs.Path.getFileSystem(Path.java:365)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWritingFileFormat(DataSource.scala:461)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWriting(DataSource.scala:558)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:390)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:363)\r\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)\r\n\tat org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:793)\r\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)\r\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:578)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\r\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\r\n\tat java.base/java.lang.Thread.run(Thread.java:1589)\r\n"
519 ]
520 }
521 ],
522 "source": [
523 "final_df.write.parquet('s3fs://shubham-test-bucket/testing/data/petco')"
524 ]
525 },
526 {
527 "cell_type": "code",
528 "execution_count": null,
529 "id": "259cae68-d414-46a8-b637-de070dbe0eb1",
530 "metadata": {},
531 "outputs": [],
532 "source": [
533 "print(datetime.datetime.now() -s )"
534 ]
535 },
536 {
537 "cell_type": "code",
538 "execution_count": null,
539 "id": "12b937d9-b40d-414c-ab8b-5404f44ad68b",
540 "metadata": {},
541 "outputs": [],
542 "source": []
543 }
544 ],
545 "metadata": {
546 "kernelspec": {
547 "display_name": "Python 3",
548 "language": "python",
549 "name": "python3"
550 },
551 "language_info": {
552 "codemirror_mode": {
553 "name": "ipython",
554 "version": 3
555 },
556 "file_extension": ".py",
557 "mimetype": "text/x-python",
558 "name": "python",
559 "nbconvert_exporter": "python",
560 "pygments_lexer": "ipython3",
561 "version": "3.7.10"
562 }
563 },
564 "nbformat": 4,
565 "nbformat_minor": 5
566 }
567