logging_bucket.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a Logging Bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "# from botocore.exceptions import ClientError\n",
    "\n",
    "import datetime\n",
    "# import pandas as pd\n",
    "# pyarrow must be installed to use parquet files in Pandas\n",
    "\n",
    "import s3_delete as s3del\n",
    "import s3_generate as s3gen\n",
    "import s3_get as s3get\n",
    "import s3_set as s3set\n",
    "\n",
    "import display_modules as dp\n",
    "# from inspect import getmembers, isfunction\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "note: sessions are not thread-safe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = boto3.session.Session(profile_name=\"default\")\n",
    "# print(\"Version: \", boto3.__version__)\n",
    "# print(\"Buckets:\", *session.client(\"s3\").list_buckets()[\"Buckets\"], sep=\"\\n  \")\n",
    "print(\"Region: \", session.region_name)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- [Create](#create)\n",
    "  - [Logging Bucket](#log)\n",
    "- [Getters](#get)\n",
    "- [Setters](#set)\n",
    "  - [Lifecycle Management](#life)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Display the Functions available in the imported modules. Remember to use the import alias not the module name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for module in [s3del, s3gen, s3get, s3set]:\n",
    "    dp.display_callables(module)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_resource = session.resource(\"s3\")\n",
    "s3_client = session.client(\"s3\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"create\"></a>\n",
    "## Create"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"log\"></a>\n",
    "### Create a Logging Bucket"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make the bucket that will be used to store the logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "creator = \"\"\n",
    "tags = s3gen.gen_tagging_list_from_python_dict({\"creator\":creator, \"content\":\"logs\"})\n",
    "logging_bucket_name, logging_bucket_response = s3gen.gen_bucket(\"logging-bucket\", tags=tags, suffix=True)\n",
    "logging_bucket_response"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grant the logging bucket the required server access logging permissions via a policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_account = \"\"\n",
    "response = s3set.grant_logging_permissions_bucket_policy(logging_bucket_name, source_account)\n",
    "response"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Turn on logging for a source bucket using the newly created logging bucket as the target bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_bucket_name = \"\"\n",
    "use_logging_bucket = \"\"\n",
    "response = s3set.set_bucket_server_access_logging_on(source_bucket_name,\n",
    "                                                     use_logging_bucket,\n",
    "                                                     logging_path_prefix=None)\n",
    "#the logging path prefix defaults to source_bucket_name if you keep it as None\n",
    "response"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"get\"></a>\n",
    "## Get S3 Things"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get list of bucket dictionaries using a prefix and/or a date or date interval as a filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3get.get_buckets_with_name_date(\"logging\", [datetime.date(2023,2,12), \"2023-2-16\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get list of bucket dictionaries using tags as filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3get.get_buckets_with_tags({\"content\":\"\"}, s3_format=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get list of object dictionaries using tags as filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3get.get_objects_with_tags_from_bucket(\"\", {\"data-type\":\"\"}, s3_format=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"set\"></a>\n",
    "## Set S3 things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_name = \"melon\"\n",
    "object_name = \"lemon\"\n",
    "creator = \"fruit-tree\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add tags to a bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = {\"creator\": creator, \"content\": \"logs\"}\n",
    "s3set.add_tags_to_bucket(bucket_name, tags, s3_format=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add tags to an object in a bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = {\"test-tag\": \"testingfunction_addtagtoobject\"}\n",
    "s3set.add_tags_to_object(bucket_name, object_name, tags, s3_format=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"life\"></a>\n",
    "### Lifecycle Management Configuration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding lifecycle management configuration policies to the logging bucket made at the beginning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_lifecycle_config = \"\"\n",
    "prefix_filter = \"\"\n",
    "# tag_filter = {}\n",
    "transition_response = s3set.add_bucket_lifecycle(\"logging-to-onezoneia-30days\",\n",
    "                           bucket_lifecycle_config,\n",
    "                           transition=\"ONEZONE_IA\",\n",
    "                           transition_days=30,\n",
    "                           expiration=False,\n",
    "                           noncurrent_expiration=True,\n",
    "                           noncurrent_expiration_days=3,\n",
    "                           prefix_filter= prefix_filter)\n",
    "expiration_response = s3set.add_bucket_lifecycle(\"logging-to-expiration-90days\",\n",
    "                           bucket_lifecycle_config,\n",
    "                           transition=None,\n",
    "                           expiration=True,\n",
    "                           expiration_days=90,\n",
    "                           noncurrent_expiration=True,\n",
    "                           noncurrent_expiration_days=3,\n",
    "                           prefix_filter=prefix_filter)\n",
    "\n",
    "print(\"\\nTransition Configuration:\\n\\t\", transition_response)\n",
    "print(\"\\nExpiration Configuration:\\n\\t\", expiration_response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tflow",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "eb10c77677f2681c8cdc430d61d84fa8c5e932d580c2651cbb47fadd398eac22"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}