{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d35cbf81-9ec9-45f7-b034-fa296d429f54", "metadata": {}, "outputs": [], "source": [ "from credentials import *\n", "from dataextractor import *\n", "from pyspark.sql import SparkSession\n", "from pyspark.sql.functions import *\n", "import datetime" ] }, { "cell_type": "code", "execution_count": 2, "id": "56454870-c431-4fe0-88dd-26fdbf3821e8", "metadata": {}, "outputs": [], "source": [ "from pyspark import SparkContext" ] }, { "cell_type": "code", "execution_count": 3, "id": "4722d484-7148-4efe-8711-0bd44027ff94", "metadata": {}, "outputs": [], "source": [ "s= datetime.datetime.now()" ] }, { "cell_type": "code", "execution_count": 4, "id": "58e514fe-b19f-485d-a368-98da34f4f021", "metadata": {}, "outputs": [], "source": [ "# from pyspark import SparkConf\n", "# conf = SparkConf().set(\"spark.hadoop.fs.s3a.access.key\", \"AKIATBUD4GB6PUTB2TOB\") \\\n", "# .set(\"spark.hadoop.fs.s3a.secret.key\", \"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\") \\\n", "# .set('packages', 'net.snowflake:snowflake-jdbc:3.13.26,net.snowflake:spark-snowflake_2.13:2.11.1-spark_3.3')" ] }, { "cell_type": "code", "execution_count": 5, "id": "0762f644-5aad-4c86-8c4f-94ae30e8c92b", "metadata": {}, "outputs": [], "source": [ "# #spark.stop()\n", "# common ='file:///c:/Users/PC/Downloads/'\n", "path1='snowflake-jdbc-3.13.26.jar'\n", "path2='spark-snowflake_2.13-2.11.1-spark_3.3.jar'\n", "p=path1+','+path2\n", "spark = SparkSession.builder \\\n", " .config('spark.jars', p) \\\n", " .appName('dev') \\\n", " .getOrCreate()" ] }, { "cell_type": "code", "execution_count": 6, "id": "b9337f3d-3553-4140-9171-93cef9ad191b", "metadata": {}, "outputs": [], "source": [ "# #spark.stop()\n", "# spark = SparkSession.builder \\\n", "# .config('packages', 'net.snowflake:snowflake-jdbc:3.13.26,net.snowflake:spark-snowflake_2.13:2.11.1-spark_3.3') \\\n", "# .config(\"spark.hadoop.fs.s3a.access.key\", \"AKIATBUD4GB6PUTB2TOB\")\\\n", "# .config(\"spark.hadoop.fs.s3a.secret.key\", \"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\")\\\n", "# .appName('dev') \\\n", "# .getOrCreate()" ] }, { "cell_type": "code", "execution_count": 7, "id": "b7dff7ae-d957-4183-9808-1618f4cc4203", "metadata": {}, "outputs": [], "source": [ "cred_manager= CredentialManager()\n", "dataextractor= DataExtractor(spark, CredentialManager())" ] }, { "cell_type": "code", "execution_count": 8, "id": "951b1951-a38e-48c5-9a11-b95b8015ba75", "metadata": {}, "outputs": [], "source": [ "options={'table':'CUSTOMER_CATALOG',\n", " 'warehouse':'COMPUTE', 'schema':'BUNGEE', 'database':'PETCO_BUNGEE_SHARE_INT'\n", " }" ] }, { "cell_type": "code", "execution_count": 9, "id": "6efa9696-85a1-4680-b69f-57f7a0fee010", "metadata": {}, "outputs": [], "source": [ "df= dataextractor.extract_data('petco-snowflake', options , 'cast(product_id as double)%2 = 0')" ] }, { "cell_type": "code", "execution_count": 10, "id": "cd909b6c-a1d1-4c0f-8a69-428d0deaa100", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "root\n", " |-- DATETIME: timestamp (nullable = false)\n", " |-- PRODUCT_ID: decimal(38,0) (nullable = false)\n", " |-- PARENT_PRODUCT_PART_DOUBLE: string (nullable = true)\n", " |-- UPC: string (nullable = true)\n", " |-- PRODUCT_NAME: string (nullable = true)\n", " |-- PRODUCT_DESCRIPTION: string (nullable = true)\n", " |-- PRODUCT_SIZE: string (nullable = true)\n", " |-- PRODUCT_COLOR: string (nullable = true)\n", " |-- PRODUCT_PURCHASE_BRAND: string (nullable = true)\n", " |-- PRIMARY_VENDOR_NAME: string (nullable = true)\n", " |-- PRODUCT_CATEGORY_LEVEL1: string (nullable = true)\n", " |-- PRODUCT_CATEGORY_LEVEL2: string (nullable = true)\n", " |-- PRODUCT_CATEGORY_LEVEL3: string (nullable = true)\n", " |-- PRODUCT_CATEGORY_LEVEL4: string (nullable = true)\n", " |-- PRODUCT_CATEGORY_LEVEL5: string (nullable = true)\n", " |-- PRODUCT_CATEGORY_LEVEL6: string (nullable = true)\n", " |-- PRODUCT_FULLIMAGE: string (nullable = true)\n", " |-- PRODUCT_URL: string (nullable = true)\n", " |-- PRODUCT_DISCONTINUED_FLAG: date (nullable = true)\n", " |-- PRODUCT_PUBLISHED_FLAG: decimal(38,0) (nullable = true)\n", " |-- PRODUCT_WEIGHT: string (nullable = true)\n", " |-- PRODUCT_WEIGHT_UOM: string (nullable = true)\n", " |-- PRODUCT_LENGTH: decimal(10,2) (nullable = true)\n", " |-- PRODUCT_WIDTH: decimal(10,2) (nullable = true)\n", " |-- PRODUCT_HEIGHT: decimal(10,2) (nullable = true)\n", "\n" ] } ], "source": [ "df.printSchema()" ] }, { "cell_type": "code", "execution_count": 11, "id": "cb044fbf-5e1a-4358-a305-f52c29b0ce2d", "metadata": {}, "outputs": [], "source": [ "grouped= df.groupby('PRODUCT_COLOR').agg(count('PRODUCT_ID').alias('count') )" ] }, { "cell_type": "code", "execution_count": 12, "id": "ce22da84-af22-4320-addf-2c898fe887b1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----------------+------+\n", "| PRODUCT_COLOR| count|\n", "+----------------+------+\n", "| null|109251|\n", "|Other or Unknown| 6179|\n", "| Multi| 6052|\n", "| Blue| 3602|\n", "| Pink| 2816|\n", "| Red| 2634|\n", "| Black| 2599|\n", "| Gray| 1967|\n", "| Brown| 1742|\n", "| Green| 1662|\n", "| White or Ivory| 1071|\n", "| Orange| 601|\n", "| Purple| 571|\n", "| Yellow| 475|\n", "| Metallic| 338|\n", "| Transparent| 67|\n", "+----------------+------+\n", "\n" ] } ], "source": [ "grouped.repartition(1).orderBy(grouped['count'].desc()).show()" ] }, { "cell_type": "code", "execution_count": 13, "id": "9972dabe-84d8-49e0-9693-a2f5d35b9f8e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----------------------+\n", "|PRODUCT_PUBLISHED_FLAG|\n", "+----------------------+\n", "| 1|\n", "| null|\n", "| 0|\n", "+----------------------+\n", "\n" ] } ], "source": [ "df.select('PRODUCT_PUBLISHED_FLAG').distinct().show()" ] }, { "cell_type": "code", "execution_count": 14, "id": "60fcc060-70ee-4dc0-b112-7f5d0c65419d", "metadata": {}, "outputs": [], "source": [ "df =df.withColumn('is_discontinued' , when(isnull(col('PRODUCT_DISCONTINUED_FLAG')), True).otherwise(False) )" ] }, { "cell_type": "code", "execution_count": 15, "id": "b66dce0c-8a2f-4e59-b329-54c86c238c80", "metadata": {}, "outputs": [], "source": [ "df =df.withColumn('is_active' , when(isnull(col('PRODUCT_DISCONTINUED_FLAG')), 0).otherwise(1) )" ] }, { "cell_type": "code", "execution_count": 16, "id": "5396ff66-7b0f-4568-82b5-0c17f4a99e29", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+---------+---------------+-----------------+\n", "|is_active|is_discontinued|count(PRODUCT_ID)|\n", "+---------+---------------+-----------------+\n", "| 0| true| 35090|\n", "| 1| false| 106537|\n", "+---------+---------------+-----------------+\n", "\n" ] } ], "source": [ "df.groupby('is_active', 'is_discontinued').agg(count(col('PRODUCT_ID'))).show()" ] }, { "cell_type": "code", "execution_count": 17, "id": "91e016f5-26ff-49ec-92cc-cf85a7a08d43", "metadata": {}, "outputs": [], "source": [ "grouped=df.filter(col('is_active')==1).groupby('PRODUCT_PURCHASE_BRAND').agg(count(col('PRODUCT_ID')).alias('count'))" ] }, { "cell_type": "code", "execution_count": 18, "id": "898ea84a-cdff-457c-b46c-c68d725c0b8d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----------------------+-----+\n", "|PRODUCT_PURCHASE_BRAND|count|\n", "+----------------------+-----+\n", "| UNASSIGNED|25060|\n", "| PETCO BRAND| 6303|\n", "| SIMPLY SHE (CUSTO...| 3313|\n", "| DICKENS CLOSET| 2947|\n", "| BOND & CO.| 1844|\n", "+----------------------+-----+\n", "only showing top 5 rows\n", "\n" ] } ], "source": [ "grouped.repartition(1).orderBy(grouped['count'].desc()).show(5)" ] }, { "cell_type": "code", "execution_count": 19, "id": "b8530fd2-7cef-4928-bde9-54d22135a87e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1830" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grouped.count()" ] }, { "cell_type": "code", "execution_count": 20, "id": "3a5be4c2-9a8b-4c2d-986c-85705284294a", "metadata": {}, "outputs": [], "source": [ "grouped2=df.filter(col('is_active')==1).groupby('PRODUCT_PURCHASE_BRAND')\\\n", " .agg(collect_set(substring(col('UPC'),1,5)).alias('upc5'),\n", " count_distinct( substring(col('UPC'),1,5)).alias('count_upc5')\n", " )" ] }, { "cell_type": "code", "execution_count": 21, "id": "697a1c98-0d40-4436-af61-7de33dd90453", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1830" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "grouped2.count()" ] }, { "cell_type": "code", "execution_count": 22, "id": "1e8c028c-ad64-475f-9c93-17a1889f0626", "metadata": {}, "outputs": [], "source": [ "final_df= grouped.repartition(1).join(grouped2.repartition(1),['PRODUCT_PURCHASE_BRAND'], 'inner')" ] }, { "cell_type": "code", "execution_count": 23, "id": "83d9dc50-131d-4ead-a1df-bde717bd266c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----------------------+-----+--------------------+----------+\n", "|PRODUCT_PURCHASE_BRAND|count| upc5|count_upc5|\n", "+----------------------+-----+--------------------+----------+\n", "| 2 FOR $5| 114|[80044, 74677, 61...| 4|\n", "| 2 FOR 5| 2| [74677]| 1|\n", "| 26 BARS AND A BAND| 57| [71812, 70510]| 2|\n", "| 3 DOG PET SUPPLY| 1| [85975]| 1|\n", "| 3M MASKING & PACK...| 39|[50051, 02120, 05...| 5|\n", "+----------------------+-----+--------------------+----------+\n", "only showing top 5 rows\n", "\n" ] } ], "source": [ "final_df.show(5)" ] }, { "cell_type": "code", "execution_count": null, "id": "9d4c489b-d2c8-4542-ad36-4a8410383908", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 24, "id": "779a43d6-b977-4152-861d-da280edbec26", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0:01:18.503431\n" ] } ], "source": [ "print(datetime.datetime.now() -s )" ] }, { "cell_type": "code", "execution_count": 25, "id": "63a0c9ac-7728-4976-82db-912d369b818a", "metadata": {}, "outputs": [], "source": [ "s=datetime.datetime.now()" ] }, { "cell_type": "code", "execution_count": 31, "id": "10555399-57e1-4de7-9841-16ab29604526", "metadata": {}, "outputs": [], "source": [ "sc = spark.sparkContext\n", "access_key='AKIATBUD4GB6PUTB2TOB'\n", "secret_key='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'\n", "# remove this block if use core-site.xml and env variable\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3.awsAccessKeyId\", access_key)\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3n.awsAccessKeyId\", access_key)\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\", access_key)\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3.awsSecretAccessKey\", secret_key)\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3n.awsSecretAccessKey\", secret_key)\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\", secret_key)\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3n.impl\", \"org.apache.hadoop.fs.s3native.NativeS3FileSystem\")\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3a.impl\", \"org.apache.hadoop.fs.s3a.S3AFileSystem\")\n", "sc._jsc.hadoopConfiguration().set(\"fs.s3.impl\", \"org.apache.hadoop.fs.s3.S3FileSystem\")" ] }, { "cell_type": "code", "execution_count": 34, "id": "6b43c7de-38b7-4e92-b6fe-48ebe4a2c483", "metadata": {}, "outputs": [ { "ename": "Py4JJavaError", "evalue": "An error occurred while calling o160.parquet.\n: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme \"s3fs\"\r\n\tat org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3443)\r\n\tat org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3466)\r\n\tat org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521)\r\n\tat org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540)\r\n\tat org.apache.hadoop.fs.Path.getFileSystem(Path.java:365)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWritingFileFormat(DataSource.scala:461)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWriting(DataSource.scala:558)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:390)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:363)\r\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)\r\n\tat org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:793)\r\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)\r\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:578)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\r\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\r\n\tat java.base/java.lang.Thread.run(Thread.java:1589)\r\n", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mPy4JJavaError\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[34], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mfinal_df\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparquet\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43ms3fs://shubham-test-bucket/testing/data/petco\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\pyspark\\sql\\readwriter.py:1140\u001b[0m, in \u001b[0;36mDataFrameWriter.parquet\u001b[1;34m(self, path, mode, partitionBy, compression)\u001b[0m\n\u001b[0;32m 1138\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpartitionBy(partitionBy)\n\u001b[0;32m 1139\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_opts(compression\u001b[38;5;241m=\u001b[39mcompression)\n\u001b[1;32m-> 1140\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jwrite\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparquet\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\py4j\\java_gateway.py:1321\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m 1315\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1316\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1317\u001b[0m args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1318\u001b[0m proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[0;32m 1320\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client\u001b[38;5;241m.\u001b[39msend_command(command)\n\u001b[1;32m-> 1321\u001b[0m return_value \u001b[38;5;241m=\u001b[39m \u001b[43mget_return_value\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 1322\u001b[0m \u001b[43m \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtarget_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1324\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n\u001b[0;32m 1325\u001b[0m temp_arg\u001b[38;5;241m.\u001b[39m_detach()\n", "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\pyspark\\sql\\utils.py:190\u001b[0m, in \u001b[0;36mcapture_sql_exception..deco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m 188\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdeco\u001b[39m(\u001b[38;5;241m*\u001b[39ma: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m 189\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 190\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 191\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m Py4JJavaError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m 192\u001b[0m converted \u001b[38;5;241m=\u001b[39m convert_exception(e\u001b[38;5;241m.\u001b[39mjava_exception)\n", "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\py4j\\protocol.py:326\u001b[0m, in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m 324\u001b[0m value \u001b[38;5;241m=\u001b[39m OUTPUT_CONVERTER[\u001b[38;5;28mtype\u001b[39m](answer[\u001b[38;5;241m2\u001b[39m:], gateway_client)\n\u001b[0;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m answer[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m==\u001b[39m REFERENCE_TYPE:\n\u001b[1;32m--> 326\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Py4JJavaError(\n\u001b[0;32m 327\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[0;32m 328\u001b[0m \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name), value)\n\u001b[0;32m 329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Py4JError(\n\u001b[0;32m 331\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m. Trace:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{3}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[0;32m 332\u001b[0m \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name, value))\n", "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling o160.parquet.\n: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme \"s3fs\"\r\n\tat org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3443)\r\n\tat org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3466)\r\n\tat org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574)\r\n\tat org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521)\r\n\tat org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540)\r\n\tat org.apache.hadoop.fs.Path.getFileSystem(Path.java:365)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWritingFileFormat(DataSource.scala:461)\r\n\tat org.apache.spark.sql.execution.datasources.DataSource.planForWriting(DataSource.scala:558)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:390)\r\n\tat org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:363)\r\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)\r\n\tat org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:793)\r\n\tat java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)\r\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:578)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:282)\r\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\r\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\r\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\r\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\r\n\tat java.base/java.lang.Thread.run(Thread.java:1589)\r\n" ] } ], "source": [ "final_df.write.parquet('s3fs://shubham-test-bucket/testing/data/petco')" ] }, { "cell_type": "code", "execution_count": null, "id": "259cae68-d414-46a8-b637-de070dbe0eb1", "metadata": {}, "outputs": [], "source": [ "print(datetime.datetime.now() -s )" ] }, { "cell_type": "code", "execution_count": null, "id": "12b937d9-b40d-414c-ab8b-5404f44ad68b", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.1" } }, "nbformat": 4, "nbformat_minor": 5 }