diff --git a/DateTimeDimensionGenerator.ipynb b/DateTimeDimensionGenerator.ipynb index c5cf5fb..1e45358 100644 --- a/DateTimeDimensionGenerator.ipynb +++ b/DateTimeDimensionGenerator.ipynb @@ -30,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 86, "metadata": { "collapsed": false }, @@ -39,14 +39,42 @@ "name": "stdout", "output_type": "stream", "text": [ - "········\n" + "Host Address: 159.203.20.161\n" ] } ], "source": [ - "host = \"ec2-52-87-185-170.compute-1.amazonaws.com\"\n", + "host = raw_input(\"Host Address: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ "db_name = \"pokemon_go\"\n", - "username = \"pokemon_go_role\"\n", + "username = \"postgres\"" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "········\n" + ] + } + ], + "source": [ "password = getpass.getpass()" ] }, @@ -732,7 +760,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 78, "metadata": { "collapsed": false }, @@ -2308,7 +2336,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 93, "metadata": { "collapsed": false }, @@ -2352,7 +2380,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 94, "metadata": { "collapsed": false }, @@ -2366,7 +2394,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 81, "metadata": { "collapsed": true }, @@ -2383,7 +2411,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 82, "metadata": { "collapsed": true }, @@ -2424,6 +2452,34 @@ "date_dim_file" ] }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'date_dim_file' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mload_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtable_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'date_dimension'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprimary_key\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"date_key\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_object\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdate_dim_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'date_dim_file' is not defined" + ] + } + ], + "source": [ + "conn = psycopg2.connect(connection_string)\n", + "conn.autocommit = True\n", + "try:\n", + " load_file(conn, table_name='date_dimension', primary_key=\"date_key\", file_object=date_dim_file)\n", + "finally:\n", + " conn.close()" + ] + }, { "cell_type": "code", "execution_count": 54, @@ -2449,23 +2505,72 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 92, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'time_dim_file' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mload_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtable_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'time_dimension'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprimary_key\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"time_key\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_object\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime_dim_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'time_dim_file' is not defined" + ] + } + ], "source": [ "conn = psycopg2.connect(connection_string)\n", "conn.autocommit = True\n", "try:\n", - " load_file(conn, table_name='date_dimension', primary_key=\"date_key\", file_object=date_dim_file)\n", + " load_file(conn, table_name='time_dimension', primary_key=\"time_key\", file_object=time_dim_file)\n", "finally:\n", " conn.close()" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 98, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "pokemon_info_df = pd.read_csv(filepath_or_buffer=\"./pokemon_info.csv\")\n", + "empty_pokemon_pk_info = pokemon_info_df.copy()\n", + "empty_pokemon_pk_info = empty_pokemon_pk_info.drop(empty_pokemon_pk_info.index[0:empty_pokemon_pk_info.shape[0]])\n", + "empty_pokemon_pk_info.to_sql(\"pokemon_info\", engine, if_exists=\"replace\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pokemon_info_dim_file = open(\"./pokemon_info.csv\")\n", + "pokemon_info_dim_file" + ] + }, + { + "cell_type": "code", + "execution_count": 100, "metadata": { "collapsed": false }, @@ -2474,7 +2579,7 @@ "conn = psycopg2.connect(connection_string)\n", "conn.autocommit = True\n", "try:\n", - " load_file(conn, table_name='time_dimension', primary_key=\"time_key\", file_object=time_dim_file)\n", + " load_file(conn, table_name='pokemon_info', primary_key=\"pokemon_id\", file_object=pokemon_info_dim_file)\n", "finally:\n", " conn.close()" ] diff --git a/README.md b/README.md index da3809d..792535b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,17 @@ A data model for doing geospatial analysis and regular analytics on Pokemon Go d ## Patches -### Jul 27, 2016 ~11PM EDT +### Jul 28, 2016 ~12PM EDT + +I've added a table for doing analysis using various Pokemon properties, such as type, classification, weight and height. To patch an existing database to support this, first drop the existing pokemon_info table from pgAdmin or using: + +```sql +DROP TABLE public.pokemon_info; +``` + +And then use the same Restore feature you used to load the database initially to load the pokemon_into_table_patch.tar file available in the repo. + +### Jul 27, 2016 ~11PM EDT If you loaded the database backup file before this time ([commit bd81308](https://github.com/Brideau/pokelyzer/commit/bd813085e0ce5518ae55e33dcc87241b710fb215)), run the following command to patch your existing database. It fixes an issue with joining tables in Tableau. diff --git a/pokemon_go_db_backup.tar b/pokemon_go_db_backup.tar index 73c35d2..f429c06 100644 Binary files a/pokemon_go_db_backup.tar and b/pokemon_go_db_backup.tar differ diff --git a/pokemon_info.csv b/pokemon_info.csv new file mode 100644 index 0000000..50423c7 --- /dev/null +++ b/pokemon_info.csv @@ -0,0 +1,152 @@ +"pokemon_id","name","classification","type_1","type_2","weight","height" +"1","Bulbasaur","Seed Pokèmon","Grass","Poison","6.9","0.7" +"2","Ivysaur","Seed Pokèmon","Grass","Poison","13.0","1.0" +"3","Venusaur","Seed Pokèmon","Grass","Poison","1.0","2.0" +"4","Charmander","Lizard Pokèmon","Fire","N/A","8.5","0.6" +"5","Charmeleon","Flame Pokèmon","Fire","N/A","19.0","1.1" +"6","Charizard","Flame Pokèmon","Fire","Flying","90.5","1.7" +"7","Squirtle","Tiny Turtle Pokèmon","Water","N/A","9.0","0.5" +"8","Wartortle","Turtle Pokèmon","Water","N/A","22.5","1.0" +"9","Blastoise","Shellfish Pokèmon","Water","N/A","85.5","1.6" +"10","Caterpie","Worm Pokèmon","Bug","N/A","2.9","0.3" +"11","Metapod","Cocoon Pokèmon","Bug","N/A","9.9","0.7" +"12","Butterfree","Butterfly Pokèmon","Bug","Flying","32.0","1.1" +"13","Weedle","Hairy Pokèmon","Bug","Poison","3.2","0.3" +"14","Kakuna","Cocoon Pokèmon","Bug","Poison","10.0","0.6" +"15","Beedrill","Poison Bee Pokèmon","Bug","Poison","29.5","1.0" +"16","Pidgey","Tiny Bird Pokèmon","Normal","Flying","1.8","0.3" +"17","Pidgeotto","Bird Pokèmon","Normal","Flying","30.0","1.1" +"18","Pidgeot","Bird Pokèmon","Normal","Flying","39.5","1.5" +"19","Rattata","Mouse Pokèmon","Normal","N/A","3.5","0.3" +"20","Raticate","Mouse Pokèmon","Normal","N/A","18.5","0.7" +"21","Spearow","Tiny Bird Pokèmon","Normal","Flying","2.0","0.3" +"22","Fearow","Beak Pokèmon","Normal","Flying","38.0","1.2" +"23","Ekans","Snake Pokèmon","Poison","N/A","6.9","2.0" +"24","Arbok","Cobra Pokèmon","Poison","N/A","65.0","3.5" +"25","Pikachu","Mouse Pokèmon","Electric","N/A","6.0","0.4" +"26","Raichu","Mouse Pokèmon","Electric","N/A","30.0","0.8" +"27","Sandshrew","Mouse Pokèmon","Ground","N/A","12.0","0.6" +"28","Sandslash","Mouse Pokèmon","Ground","N/A","29.5","1.0" +"29","Nidoran F","Poison Pin Pokèmon","Poison","N/A","7.0","0.4" +"30","Nidorina","Poison Pin Pokèmon","Poison","N/A","20.0","0.8" +"31","Nidoqueen","Drill Pokèmon","Poison","Ground","60.0","1.3" +"32","Nidoran M","Poison Pin Pokèmon","Poison","N/A","9.0","0.5" +"33","Nidorino","Poison Pin Pokèmon","Poison","N/A","19.5","0.9" +"34","Nidoking","Drill Pokèmon","Poison","Ground","62.0","1.4" +"35","Clefairy","Fairy Pokèmon","Normal","N/A","7.5","0.6" +"36","Clefable","Fairy Pokèmon","Normal","N/A","40.0","1.3" +"37","Vulpix","Fox Pokèmon","Fire","N/A","9.9","0.6" +"38","Ninetales","Fox Pokèmon","Fire","N/A","19.9","1.1" +"39","Jigglypuff","Balloon Pokèmon","Normal","N/A","5.5","0.5" +"40","Wigglytuff","Balloon Pokèmon","Normal","N/A","12.0","1.0" +"41","Zubat","Bat Pokèmon","Poison","Flying","7.5","0.8" +"42","Golbat","Bat Pokèmon","Poison","Flying","55.0","1.6" +"43","Oddish","Weed Pokèmon","Grass","Poison","5.4","0.5" +"44","Gloom","Weed Pokèmon","Grass","Poison","8.6","0.8" +"45","Vileplume","Flower Pokèmon","Grass","Poison","18.6","1.2" +"46","Paras","Mushroom Pokèmon","Bug","Grass","5.4","0.3" +"47","Parasect","Mushroom Pokèmon","Bug","Grass","29.5","1.0" +"48","Venonat","Insect Pokèmon","Bug","Poison","30.0","1.0" +"49","Venomoth","Poison Moth Pokèmon","Bug","Poison","12.5","1.5" +"50","Diglett","Mole Pokèmon","Ground","N/A","0.8","0.2" +"51","Dugtrio","Mole Pokèmon","Ground","N/A","33.3","0.7" +"52","Meowth","Scratch Cat Pokèmon","Normal","N/A","4.2","0.4" +"53","Persian","Classy Cat Pokèmon","Normal","N/A","32.0","1.0" +"54","Psyduck","Duck Pokèmon","Water","N/A","19.6","0.8" +"55","Golduck","Duck Pokèmon","Water","N/A","76.6","1.7" +"56","Mankey","Pig Monkey Pokèmon","Fighting","N/A","28.0","0.5" +"57","Primeape","Pig Monkey Pokèmon","Fighting","N/A","32.0","1.0" +"58","Growlithe","Puppy Pokèmon","Fire","N/A","19.0","0.7" +"59","Arcanine","Legendary Pokèmon","Fire","N/A","155.0","1.9" +"60","Poliwag","Tadpole Pokèmon","Water","N/A","12.4","0.6" +"61","Poliwhirl","Tadpole Pokèmon","Water","N/A","20.0","1.0" +"62","Poliwrath","Tadpole Pokèmon","Water","Fighting","54.0","1.3" +"63","Abra","Psi Pokèmon","Psychic","N/A","19.5","0.9" +"64","Kadabra","Psi Pokèmon","Psychic","N/A","56.5","1.3" +"65","Alakazam","Psi Pokèmon","Psychic","N/A","48.0","1.5" +"66","Machop","Superpower Pokèmon","Fighting","N/A","19.5","0.8" +"67","Machoke","Superpower Pokèmon","Fighting","N/A","70.5","1.5" +"68","Machamp","Superpower Pokèmon","Fighting","N/A","130.0","1.6" +"69","Bellsprout","Flower Pokèmon","Grass","Poison","4.0","0.7" +"70","Weepinbell","Flycatcher Pokèmon","Grass","Poison","6.4","1.0" +"71","Victreebel","Flycatcher Pokèmon","Grass","Poison","15.5","1.7" +"72","Tentacool","Jellyfish Pokèmon","Water","Poison","45.5","0.9" +"73","Tentacruel","Jellyfish Pokèmon","Water","Poison","55.0","1.6" +"74","Geodude","Rock Pokèmon","Rock","Ground","20.0","0.4" +"75","Graveler","Rock Pokèmon","Rock","Ground","105.0","1.0" +"76","Golem","Megaton Pokèmon","Rock","Ground","300.0","1.4" +"77","Ponyta","Fire Horse Pokèmon","Fire","N/A","30.0","1.0" +"78","Rapidash","Fire Horse Pokèmon","Fire","N/A","95.0","1.7" +"79","Slowpoke","Dopey Pokèmon","Water","Psychic","36.0","1.2" +"80","Slowbro","Hermit Crab Pokèmon","Water","Psychic","78.5","1.6" +"81","Magnemite","Magnet Pokèmon","Electric","Steel","6.0","0.3" +"82","Magneton","Magnet Pokèmon","Electric","Steel","60.0","1.0" +"83","Farfetch'd","Wild Duck Pokèmon","Normal","Flying","15.0","0.8" +"84","Doduo","Twin Bird Pokèmon","Normal","Flying","39.2","1.4" +"85","Dodrio","Triple Bird Pokèmon","Normal","Flying","85.2","1.8" +"86","Seel","Sea Lion Pokèmon","Water","N/A","90.0","1.1" +"87","Dewgong","Sea Lion Pokèmon","Water","Ice","120.0","1.7" +"88","Grimer","Sludge Pokèmon","Poison","N/A","30.0","0.9" +"89","Muk","Sludge Pokèmon","Poison","N/A","30.0","1.2" +"90","Shellder","Bivalve Pokèmon","Water","N/A","4.0","0.3" +"91","Cloyster","Bivalve Pokèmon","Water","Ice","132.5","1.5" +"92","Gastly","Gas Pokèmon","Ghost","Poison","0.1","1.3" +"93","Haunter","Gas Pokèmon","Ghost","Poison","0.1","1.6" +"94","Gengar","Shadow Pokèmon","Ghost","Poison","40.5","1.5" +"95","Onix","Rock Snake Pokèmon","Rock","Ground","210.0","8.8" +"96","Drowzee","Hypnosis Pokèmon","Psychic","N/A","32.4","1.0" +"97","Hypno","Hypnosis Pokèmon","Psychic","N/A","75.6","1.6" +"98","Krabby","River Crab Pokèmon","Water","N/A","6.5","0.4" +"99","Kingler","Pincer Pokèmon","Water","N/A","60.0","1.3" +"100","Voltorb","Ball Pokèmon","Electric","N/A","10.4","0.5" +"101","Electrode","Ball Pokèmon","Electric","N/A","66.6","1.2" +"102","Exeggcute","Egg Pokèmon","Grass","Psychic","2.5","0.4" +"103","Exeggutor","Coconut Pokèmon","Grass","Psychic","120.0","2.0" +"104","Cubone","Lonely Pokèmon","Ground","N/A","6.5","0.4" +"105","Marowak","Bone Keeper Pokèmon","Ground","N/A","45.0","1.0" +"106","Hitmonlee","Kicking Pokèmon","Fighting","N/A","49.8","1.5" +"107","Hitmonchan","Punching Pokèmon","Fighting","N/A","50.2","1.4" +"108","Lickitung","Licking Pokèmon","Normal","N/A","65.5","1.2" +"109","Koffing","Poison Gas Pokèmon","Poison","N/A","1.0","0.6" +"110","Weezing","Poison Gas Pokèmon","Poison","N/A","9.5","1.2" +"111","Rhyhorn","Spikes Pokèmon","Ground","Rock","115.0","1.0" +"112","Rhydon","Drill Pokèmon","Ground","Rock","120.0","1.9" +"113","Chansey","Egg Pokèmon","Normal","N/A","34.6","1.1" +"114","Tangela","Vine Pokèmon","Grass","N/A","35.0","1.0" +"115","Kangaskhan","Parent Pokèmon","Normal","N/A","80.0","2.2" +"116","Horsea","Dragon Pokèmon","Water","N/A","8.0","0.4" +"117","Seadra","Dragon Pokèmon","Water","N/A","25.0","1.2" +"118","Goldeen","Goldfish Pokèmon","Water","N/A","15.0","0.6" +"119","Seaking","Goldfish Pokèmon","Water","N/A","39.0","1.3" +"120","Staryu","Starshape Pokèmon","Water","N/A","34.5","0.8" +"121","Starmie","Mysterious Pokèmon","Water","Psychic","80.0","1.1" +"122","Mr. Mime","Barrier Pokèmon","Psychic","N/A","54.5","1.3" +"123","Scyther","Mantis Pokèmon","Bug","Flying","56.0","1.5" +"124","Jynx","Humanshape Pokèmon","Ice","Psychic","40.6","1.4" +"125","Electabuzz","Electric Pokèmon","Electric","N/A","30.0","1.1" +"126","Magmar","Spitfire Pokèmon","Fire","N/A","44.5","1.3" +"127","Pinsir","Stagbeetle Pokèmon","Bug","N/A","55.0","1.5" +"128","Tauros","Wild Bull Pokèmon","Normal","N/A","88.4","1.4" +"129","Magikarp","Fish Pokèmon","Water","N/A","10.0","0.9" +"130","Gyarados","Atrocious Pokèmon","Water","Flying","235.0","6.5" +"131","Lapras","Transport Pokèmon","Water","Ice","220.0","2.5" +"132","Ditto","Transform Pokèmon","Normal","N/A","4.0","0.3" +"133","Eevee","Evolution Pokèmon","Normal","N/A","6.5","0.3" +"134","Vaporeon","Bubble Jet Pokèmon","Water","N/A","29.0","1.0" +"135","Jolteon","Lightning Pokèmon","Electric","N/A","24.5","0.8" +"136","Flareon","Flame Pokèmon","Fire","N/A","25.0","0.9" +"137","Porygon","Virtual Pokèmon","Normal","N/A","36.5","0.8" +"138","Omanyte","Spiral Pokèmon","Rock","Water","7.5","0.4" +"139","Omastar","Spiral Pokèmon","Rock","Water","35.0","1.0" +"140","Kabuto","Shellfish Pokèmon","Rock","Water","11.5","0.5" +"141","Kabutops","Shellfish Pokèmon","Rock","Water","40.5","1.3" +"142","Aerodactyl","Fossil Pokèmon","Rock","Flying","59.0","1.8" +"143","Snorlax","Sleeping Pokèmon","Normal","N/A","460.0","2.1" +"144","Articuno","Freeze Pokèmon","Ice","Flying","55.4","1.7" +"145","Zapdos","Electric Pokèmon","Electric","Flying","52.6","1.6" +"146","Moltres","Flame Pokèmon","Fire","Flying","60.0","2.0" +"147","Dratini","Dragon Pokèmon","Dragon","N/A","3.3","1.8" +"148","Dragonair","Dragon Pokèmon","Dragon","N/A","16.5","4.0" +"149","Dragonite","Dragon Pokèmon","Dragon","Flying","210.0","2.2" +"150","Mewtwo","Genetic Pokèmon","Psychic","N/A","122.0","2.0" +"151","Mew","New Species Pokèmon","Psychic","N/A","4.0","0.4" diff --git a/pokemon_info_table_patch.tar b/pokemon_info_table_patch.tar new file mode 100644 index 0000000..b9e0a0d Binary files /dev/null and b/pokemon_info_table_patch.tar differ