sem/CVML-MachineLearning
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add all files

Browse Source
This commit is contained in:
Sem van der Hoeven
2021-05-26 15:12:05 +02:00
parent 5c3be10247
commit d979ca38f5
44 changed files with 43574 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

404
solutions/2 Data exploration Exercises Solution.ipynb Normal file
View File

@@ -0,0 +1,404 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import pandas as pd"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 1\n",
"- load the dataset: `../data/international-airline-passengers.csv`\n",
"- inspect it using the `.info()` and `.head()` commands\n",
"- use the function `pd.to_datetime()` to change the column type of 'Month' to a datatime type\n",
"- set the index of df to be a datetime index using the column 'Month' and the `df.set_index()` method\n",
"- choose the appropriate plot and display the data\n",
"- choose appropriate scale\n",
"- label the axes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - load the dataset: ../data/international-airline-passengers.csv\n",
"df = pd.read_csv('../data/international-airline-passengers.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - inspect it using the .info() and .head() commands\n",
"df.info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - use the function to_datetime() to change the column type of 'Month' to a datatime type\n",
"# - set the index of df to be a datetime index using the column 'Month' and tthe set_index() method\n",
"\n",
"df['Month'] = pd.to_datetime(df['Month'])\n",
"df = df.set_index('Month')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - choose the appropriate plot and display the data\n",
"# - choose appropriate scale\n",
"# - label the axes\n",
"\n",
"df.plot();"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 2\n",
"- load the dataset: `../data/weight-height.csv`\n",
"- inspect it\n",
"- plot it using a scatter plot with Weight as a function of Height\n",
"- plot the male and female populations with 2 different colors on a new scatter plot\n",
"- remember to label the axes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - load the dataset: ../data/weight-height.csv\n",
"# - inspect it\n",
"df = pd.read_csv('../data/weight-height.csv')\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.describe()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"df['Gender'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - plot it using a scatter plot with Weight as a function of Height\n",
"_ = df.plot(kind='scatter', x='Height', y='Weight');"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# - plot the male and female populations with 2 different colors on a new scatter plot\n",
"# - remember to label the axes\n",
"\n",
"# this can be done in several ways, showing 2 here:\n",
"males = df[df['Gender'] == 'Male']\n",
"females = df.query('Gender == \"Female\"')\n",
"fig, ax = plt.subplots()\n",
"\n",
"males.plot(kind='scatter', x='Height', y='Weight',\n",
" ax=ax, color='blue', alpha=0.3,\n",
" title='Male & Female Populations')\n",
"\n",
"females.plot(kind='scatter', x='Height', y='Weight',\n",
" ax=ax, color='red', alpha=0.3);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df['Gendercolor'] = df['Gender'].map({'Male': 'blue', 'Female': 'red'})\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.plot(kind='scatter', \n",
" x='Height',\n",
" y='Weight',\n",
" c=df['Gendercolor'],\n",
" alpha=0.3,\n",
" title='Male & Female Populations');"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig, ax = plt.subplots()\n",
"ax.plot(males['Height'], males['Weight'], 'ob', \n",
" females['Height'], females['Weight'], 'or', alpha=0.3)\n",
"plt.xlabel('Height')\n",
"plt.ylabel('Weight')\n",
"plt.title('Male & Female Populations');"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"## Exercise 3\n",
"- plot the histogram of the heights for males and for females on the same plot\n",
"- use alpha to control transparency in the plot comand\n",
"- plot a vertical line at the mean of each population using `plt.axvline()`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"males['Height'].plot(kind='hist',\n",
" bins=50,\n",
" range=(50, 80),\n",
" alpha=0.3,\n",
" color='blue')\n",
"\n",
"females['Height'].plot(kind='hist',\n",
" bins=50,\n",
" range=(50, 80),\n",
" alpha=0.3,\n",
" color='red')\n",
"\n",
"plt.title('Height distribution')\n",
"plt.legend([\"Males\", \"Females\"])\n",
"plt.xlabel(\"Heigth (in)\")\n",
"\n",
"\n",
"plt.axvline(males['Height'].mean(), color='blue', linewidth=2)\n",
"plt.axvline(females['Height'].mean(), color='red', linewidth=2);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"males['Height'].plot(kind='hist',\n",
" bins=200,\n",
" range=(50, 80),\n",
" alpha=0.3,\n",
" color='blue',\n",
" cumulative=True,\n",
" density=True)\n",
"\n",
"females['Height'].plot(kind='hist',\n",
" bins=200,\n",
" range=(50, 80),\n",
" alpha=0.3,\n",
" color='red',\n",
" cumulative=True,\n",
" density=True)\n",
"\n",
"plt.title('Height distribution')\n",
"plt.legend([\"Males\", \"Females\"])\n",
"plt.xlabel(\"Heigth (in)\")\n",
"\n",
"plt.axhline(0.8)\n",
"plt.axhline(0.5)\n",
"plt.axhline(0.2);"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 4\n",
"- plot the weights of the males and females using a box plot\n",
"- which one is easier to read?\n",
"- (remember to put in titles, axes and legends)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dfpvt = df.pivot(columns = 'Gender', values = 'Weight')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dfpvt.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dfpvt.info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dfpvt.plot(kind='box')\n",
"plt.title('Weight Box Plot')\n",
"plt.ylabel(\"Weight (lbs)\");"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 5\n",
"- load the dataset: `../data/titanic-train.csv`\n",
"- learn about scattermatrix here: http://pandas.pydata.org/pandas-docs/stable/visualization.html\n",
"- display the data using a scattermatrix"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv('../data/titanic-train.csv')\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pandas.plotting import scatter_matrix"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"_ = scatter_matrix(df.drop('PassengerId', axis=1), figsize=(10, 10))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

669
solutions/3 Machine Learning Exercises Solution.ipynb Normal file
View File

@@ -0,0 +1,669 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Machine Learning Exercises Solution"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 1\n",
"\n",
"You've just been hired at a real estate investment firm and they would like you to build a model for pricing houses. You are given a dataset that contains data for house prices and a few features like number of bedrooms, size in square feet and age of the house. Let's see if you can build a model that is able to predict the price. In this exercise we extend what we have learned about linear regression to a dataset with more than one feature. Here are the steps to complete it:\n",
"\n",
"1. Load the dataset ../data/housing-data.csv\n",
"- plot the histograms for each feature\n",
"- create 2 variables called X and y: X shall be a matrix with 3 columns (sqft,bdrms,age) and y shall be a vector with 1 column (price)\n",
"- create a linear regression model in Keras with the appropriate number of inputs and output\n",
"- split the data into train and test with a 20% test size\n",
"- train the model on the training set and check its accuracy on training and test set\n",
"- how's your model doing? Is the loss growing smaller?\n",
"- try to improve your model with these experiments:\n",
" - normalize the input features with one of the rescaling techniques mentioned above\n",
" - use a different value for the learning rate of your model\n",
" - use a different optimizer\n",
"- once you're satisfied with training, check the R2score on the test set"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Load the dataset ../data/housing-data.csv\n",
"df = pd.read_csv('../data/housing-data.csv')\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.columns"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# plot the histograms for each feature\n",
"plt.figure(figsize=(15, 5))\n",
"for i, feature in enumerate(df.columns):\n",
" plt.subplot(1, 4, i+1)\n",
" df[feature].plot(kind='hist', title=feature)\n",
" plt.xlabel(feature)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# create 2 variables called X and y:\n",
"# X shall be a matrix with 3 columns (sqft,bdrms,age)\n",
"# and y shall be a vector with 1 column (price)\n",
"X = df[['sqft', 'bdrms', 'age']].values\n",
"y = df['price'].values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dense\n",
"from tensorflow.keras.optimizers import Adam"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# create a linear regression model in Keras\n",
"# with the appropriate number of inputs and output\n",
"model = Sequential()\n",
"model.add(Dense(1, input_shape=(3,)))\n",
"model.compile(Adam(learning_rate=0.8), 'mean_squared_error')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# split the data into train and test with a 20% test size\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"len(X_train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"len(X)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# train the model on the training set and check its accuracy on training and test set\n",
"# how's your model doing? Is the loss growing smaller?\n",
"model.fit(X_train, y_train, epochs=10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.describe()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import r2_score"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# check the R2score on training and test set (probably very bad)\n",
"\n",
"y_train_pred = model.predict(X_train)\n",
"y_test_pred = model.predict(X_test)\n",
"\n",
"print(\"The R2 score on the Train set is:\\t{:0.3f}\".format(r2_score(y_train, y_train_pred)))\n",
"print(\"The R2 score on the Test set is:\\t{:0.3f}\".format(r2_score(y_test, y_test_pred)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# try to improve your model with these experiments:\n",
"# - normalize the input features with one of the rescaling techniques mentioned above\n",
"# - use a different value for the learning rate of your model\n",
"# - use a different optimizer\n",
"df['sqft1000'] = df['sqft']/1000.0\n",
"df['age10'] = df['age']/10.0\n",
"df['price100k'] = df['price']/1e5"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X = df[['sqft1000', 'bdrms', 'age10']].values\n",
"y = df['price100k'].values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
"model.add(Dense(1, input_dim=3))\n",
"model.compile(Adam(learning_rate=0.1), 'mean_squared_error')\n",
"model.fit(X_train, y_train, epochs=20)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# once you're satisfied with training, check the R2score on the test set\n",
"\n",
"y_train_pred = model.predict(X_train)\n",
"y_test_pred = model.predict(X_test)\n",
"\n",
"print(\"The R2 score on the Train set is:\\t{:0.3f}\".format(r2_score(y_train, y_train_pred)))\n",
"print(\"The R2 score on the Test set is:\\t{:0.3f}\".format(r2_score(y_test, y_test_pred)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train, y_train, epochs=40, verbose=0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# once you're satisfied with training, check the R2score on the test set\n",
"\n",
"y_train_pred = model.predict(X_train)\n",
"y_test_pred = model.predict(X_test)\n",
"\n",
"print(\"The R2 score on the Train set is:\\t{:0.3f}\".format(r2_score(y_train, y_train_pred)))\n",
"print(\"The R2 score on the Test set is:\\t{:0.3f}\".format(r2_score(y_test, y_test_pred)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 2\n",
"\n",
"Your boss was extremely happy with your work on the housing price prediction model and decided to entrust you with a more challenging task. They've seen a lot of people leave the company recently and they would like to understand why that's happening. They have collected historical data on employees and they would like you to build a model that is able to predict which employee will leave next. The would like a model that is better than random guessing. They also prefer false negatives than false positives, in this first phase. Fields in the dataset include:\n",
"\n",
"- Employee satisfaction level\n",
"- Last evaluation\n",
"- Number of projects\n",
"- Average monthly hours\n",
"- Time spent at the company\n",
"- Whether they have had a work accident\n",
"- Whether they have had a promotion in the last 5 years\n",
"- Department\n",
"- Salary\n",
"- Whether the employee has left\n",
"\n",
"Your goal is to predict the binary outcome variable `left` using the rest of the data. Since the outcome is binary, this is a classification problem. Here are some things you may want to try out:\n",
"\n",
"1. load the dataset at ../data/HR_comma_sep.csv, inspect it with `.head()`, `.info()` and `.describe()`.\n",
"- Establish a benchmark: what would be your accuracy score if you predicted everyone stay?\n",
"- Check if any feature needs rescaling. You may plot a histogram of the feature to decide which rescaling method is more appropriate.\n",
"- convert the categorical features into binary dummy columns. You will then have to combine them with the numerical features using `pd.concat`.\n",
"- do the usual train/test split with a 20% test size\n",
"- play around with learning rate and optimizer\n",
"- check the confusion matrix, precision and recall\n",
"- check if you still get the same results if you use a 5-Fold cross validation on all the data\n",
"- Is the model good enough for your boss?\n",
"\n",
"As you will see in this exercise, the a logistic regression model is not good enough to help your boss. In the next chapter we will learn how to go beyond linear models.\n",
"\n",
"This dataset comes from https://www.kaggle.com/ludobenistant/hr-analytics/ and is released under [CC BY-SA 4.0 License](https://creativecommons.org/licenses/by-sa/4.0/)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# load the dataset at ../data/HR_comma_sep.csv, inspect it with `.head()`, `.info()` and `.describe()`.\n",
"\n",
"df = pd.read_csv('../data/HR_comma_sep.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.describe()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Establish a benchmark: what would be your accuracy score if you predicted everyone stay?\n",
"\n",
"df.left.value_counts() / len(df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Predicting 0 all the time would yield an accuracy of 76%"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Check if any feature needs rescaling.\n",
"# You may plot a histogram of the feature to decide which rescaling method is more appropriate.\n",
"df['average_montly_hours'].plot(kind='hist');"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df['average_montly_hours_100'] = df['average_montly_hours']/100.0"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df['average_montly_hours_100'].plot(kind='hist');"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df['time_spend_company'].plot(kind='hist');"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# convert the categorical features into binary dummy columns.\n",
"# You will then have to combine them with\n",
"# the numerical features using `pd.concat`.\n",
"df_dummies = pd.get_dummies(df[['sales', 'salary']])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_dummies.head()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.columns"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X = pd.concat([df[['satisfaction_level', 'last_evaluation', 'number_project',\n",
" 'time_spend_company', 'Work_accident',\n",
" 'promotion_last_5years', 'average_montly_hours_100']],\n",
" df_dummies], axis=1).values\n",
"y = df['left'].values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# do the usual train/test split with a 20% test size\n",
"\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# play around with learning rate and optimizer\n",
"\n",
"model = Sequential()\n",
"model.add(Dense(1, input_dim=20, activation='sigmoid'))\n",
"model.compile(Adam(learning_rate=0.5), 'binary_crossentropy', metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train, y_train, epochs=10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_test_pred = model.predict_classes(X_test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import confusion_matrix, classification_report"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def pretty_confusion_matrix(y_true, y_pred, labels=[\"False\", \"True\"]):\n",
" cm = confusion_matrix(y_true, y_pred)\n",
" pred_labels = ['Predicted '+ l for l in labels]\n",
" df = pd.DataFrame(cm, index=labels, columns=pred_labels)\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# check the confusion matrix, precision and recall\n",
"\n",
"pretty_confusion_matrix(y_test, y_test_pred, labels=['Stay', 'Leave'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(classification_report(y_test, y_test_pred))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.wrappers.scikit_learn import KerasClassifier"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# check if you still get the same results if you use a 5-Fold cross validation on all the data\n",
"\n",
"def build_logistic_regression_model():\n",
" model = Sequential()\n",
" model.add(Dense(1, input_dim=20, activation='sigmoid'))\n",
" model.compile(Adam(learning_rate=0.5), 'binary_crossentropy', metrics=['accuracy'])\n",
" return model\n",
"\n",
"model = KerasClassifier(build_fn=build_logistic_regression_model,\n",
" epochs=10, verbose=0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import KFold, cross_val_score"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cv = KFold(5, shuffle=True)\n",
"scores = cross_val_score(model, X, y, cv=cv)\n",
"\n",
"print(\"The cross validation accuracy is {:0.4f} ± {:0.4f}\".format(scores.mean(), scores.std()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Is the model good enough for your boss?"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"No, the model is not good enough for my boss, since it performs no better than the benchmark."
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

413
solutions/4 Deep Learning Intro Exercises Solution.ipynb Normal file
View File

@@ -0,0 +1,413 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Deep Learning Intro"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 1"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"The [Pima Indians dataset](https://archive.ics.uci.edu/ml/datasets/diabetes) is a very famous dataset distributed by UCI and originally collected from the National Institute of Diabetes and Digestive and Kidney Diseases. It contains data from clinical exams for women age 21 and above of Pima indian origins. The objective is to predict based on diagnostic measurements whether a patient has diabetes.\n",
"\n",
"It has the following features:\n",
"\n",
"- Pregnancies: Number of times pregnant\n",
"- Glucose: Plasma glucose concentration a 2 hours in an oral glucose tolerance test\n",
"- BloodPressure: Diastolic blood pressure (mm Hg)\n",
"- SkinThickness: Triceps skin fold thickness (mm)\n",
"- Insulin: 2-Hour serum insulin (mu U/ml)\n",
"- BMI: Body mass index (weight in kg/(height in m)^2)\n",
"- DiabetesPedigreeFunction: Diabetes pedigree function\n",
"- Age: Age (years)\n",
"\n",
"The last colum is the outcome, and it is a binary variable.\n",
"\n",
"In this first exercise we will explore it through the following steps:\n",
"\n",
"1. Load the ..data/diabetes.csv dataset, use pandas to explore the range of each feature\n",
"- For each feature draw a histogram. Bonus points if you draw all the histograms in the same figure.\n",
"- Explore correlations of features with the outcome column. You can do this in several ways, for example using the `sns.pairplot` we used above or drawing a heatmap of the correlations.\n",
"- Do features need standardization? If so what stardardization technique will you use? MinMax? Standard?\n",
"- Prepare your final `X` and `y` variables to be used by a ML model. Make sure you define your target variable well. Will you need dummy columns?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv('../data/diabetes.csv')\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"_ = df.hist(figsize=(12, 10))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import seaborn as sns"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sns.pairplot(df, hue='Outcome');"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sns.heatmap(df.corr(), annot = True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.describe()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import StandardScaler"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.utils import to_categorical"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sc = StandardScaler()\n",
"X = sc.fit_transform(df.drop('Outcome', axis=1))\n",
"y = df['Outcome'].values\n",
"y_cat = to_categorical(y)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_cat.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 2"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"Build a fully connected NN model that predicts diabetes. Follow these steps:\n",
"\n",
"1. Split your data in a train/test with a test size of 20% and a `random_state = 22`\n",
"- define a sequential model with at least one inner layer. You will have to make choices for the following things:\n",
" - what is the size of the input?\n",
" - how many nodes will you use in each layer?\n",
" - what is the size of the output?\n",
" - what activation functions will you use in the inner layers?\n",
" - what activation function will you use at output?\n",
" - what loss function will you use?\n",
" - what optimizer will you use?\n",
"- fit your model on the training set, using a validation_split of 0.1\n",
"- test your trained model on the test data from the train/test split\n",
"- check the accuracy score, the confusion matrix and the classification report"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(X, y_cat,\n",
" random_state=22,\n",
" test_size=0.2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dense\n",
"from tensorflow.keras.optimizers import Adam"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
"model.add(Dense(32, input_shape=(8,), activation='relu'))\n",
"model.add(Dense(32, activation='relu'))\n",
"model.add(Dense(2, activation='softmax'))\n",
"model.compile(Adam(learning_rate=0.05),\n",
" loss='categorical_crossentropy',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"32*8 + 32"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train, y_train, epochs=20, verbose=2, validation_split=0.1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_pred = model.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_test_class = np.argmax(y_test, axis=1)\n",
"y_pred_class = np.argmax(y_pred, axis=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score\n",
"from sklearn.metrics import classification_report\n",
"from sklearn.metrics import confusion_matrix"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pd.Series(y_test_class).value_counts() / len(y_test_class)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"accuracy_score(y_test_class, y_pred_class)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(classification_report(y_test_class, y_pred_class))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"confusion_matrix(y_test_class, y_pred_class)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 3\n",
"Compare your work with the results presented in [this notebook](https://www.kaggle.com/sheshu/pima-data-visualisation-and-machine-learning). Are your Neural Network results better or worse than the results obtained by traditional Machine Learning techniques?\n",
"\n",
"- Try training a Support Vector Machine or a Random Forest model on the exact same train/test split. Is the performance better or worse?\n",
"- Try restricting your features to only 4 features like in the suggested notebook. How does model performance change?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import RandomForestClassifier\n",
"from sklearn.svm import SVC\n",
"from sklearn.naive_bayes import GaussianNB\n",
"\n",
"for mod in [RandomForestClassifier(), SVC(), GaussianNB()]:\n",
" mod.fit(X_train, y_train[:, 1])\n",
" y_pred = mod.predict(X_test)\n",
" print(\"=\"*80)\n",
" print(mod)\n",
" print(\"-\"*80)\n",
" print(\"Accuracy score: {:0.3}\".format(accuracy_score(y_test_class,\n",
" y_pred)))\n",
" print(\"Confusion Matrix:\")\n",
" print(confusion_matrix(y_test_class, y_pred))\n",
" print()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 4\n",
"\n",
"[Tensorflow playground](http://playground.tensorflow.org/) is a web based neural network demo. It is really useful to develop an intuition about what happens when you change architecture, activation function or other parameters. Try playing with it for a few minutes. You don't nee do understand the meaning of every knob and button in the page, just get a sense for what happens if you change something. In the next chapter we'll explore these things in more detail.\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

518
solutions/5 Gradient Descent Exercises Solution.ipynb Normal file
View File

@@ -0,0 +1,518 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Gradient Descent"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"%matplotlib inline\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Exercise 1\n",
"\n",
"You've just been hired at a wine company and they would like you to help them build a model that predicts the quality of their wine based on several measurements. They give you a dataset with wine\n",
"\n",
"- Load the ../data/wines.csv into Pandas\n",
"- Use the column called \"Class\" as target\n",
"- Check how many classes are there in target, and if necessary use dummy columns for a multi-class classification\n",
"- Use all the other columns as features, check their range and distribution (using seaborn pairplot)\n",
"- Rescale all the features using either MinMaxScaler or StandardScaler\n",
"- Build a deep model with at least 1 hidden layer to classify the data\n",
"- Choose the cost function, what will you use? Mean Squared Error? Binary Cross-Entropy? Categorical Cross-Entropy?\n",
"- Choose an optimizer\n",
"- Choose a value for the learning rate, you may want to try with several values\n",
"- Choose a batch size\n",
"- Train your model on all the data using a `validation_split=0.2`. Can you converge to 100% validation accuracy?\n",
"- What's the minumum number of epochs to converge?\n",
"- Repeat the training several times to verify how stable your results are"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv('../data/wines.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y = df['Class']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y.value_counts()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_cat = pd.get_dummies(y)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_cat.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X = df.drop('Class', axis=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import seaborn as sns"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sns.pairplot(df, hue='Class')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import StandardScaler"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sc = StandardScaler()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"Xsc = sc.fit_transform(X)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dense\n",
"from tensorflow.keras.optimizers import SGD, Adam, Adadelta, RMSprop\n",
"import tensorflow.keras.backend as K"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"model = Sequential()\n",
"model.add(Dense(5, input_shape=(13,),\n",
" kernel_initializer='he_normal',\n",
" activation='relu'))\n",
"model.add(Dense(3, activation='softmax'))\n",
"\n",
"model.compile(RMSprop(learning_rate=0.1),\n",
" 'categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"\n",
"model.fit(Xsc, y_cat.values,\n",
" batch_size=8,\n",
" epochs=10,\n",
" verbose=1,\n",
" validation_split=0.2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Exercise 2\n",
"\n",
"Since this dataset has 13 features we can only visualize pairs of features like we did in the Paired plot. We could however exploit the fact that a neural network is a function to extract 2 high level features to represent our data.\n",
"\n",
"- Build a deep fully connected network with the following structure:\n",
" - Layer 1: 8 nodes\n",
" - Layer 2: 5 nodes\n",
" - Layer 3: 2 nodes\n",
" - Output : 3 nodes\n",
"- Choose activation functions, inizializations, optimizer and learning rate so that it converges to 100% accuracy within 20 epochs (not easy)\n",
"- Remember to train the model on the scaled data\n",
"- Define a Feature Function like we did above between the input of the 1st layer and the output of the 3rd layer\n",
"- Calculate the features and plot them on a 2-dimensional scatter plot\n",
"- Can we distinguish the 3 classes well?\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"model = Sequential()\n",
"model.add(Dense(8, input_shape=(13,),\n",
" kernel_initializer='he_normal', activation='tanh'))\n",
"model.add(Dense(5, kernel_initializer='he_normal', activation='tanh'))\n",
"model.add(Dense(2, kernel_initializer='he_normal', activation='tanh'))\n",
"model.add(Dense(3, activation='softmax'))\n",
"\n",
"model.compile(RMSprop(learning_rate=0.05),\n",
" 'categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"\n",
"model.fit(Xsc, y_cat.values,\n",
" batch_size=16,\n",
" epochs=20,\n",
" verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"inp = model.layers[0].input\n",
"out = model.layers[2].output"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"features_function = K.function([inp], [out])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"features = features_function([Xsc])[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"features.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.scatter(features[:, 0], features[:, 1], c=y)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Exercise 3\n",
"\n",
"Keras functional API. So far we've always used the Sequential model API in Keras. However, Keras also offers a Functional API, which is much more powerful. You can find its [documentation here](https://keras.io/getting-started/functional-api-guide/). Let's see how we can leverage it.\n",
"\n",
"- define an input layer called `inputs`\n",
"- define two hidden layers as before, one with 8 nodes, one with 5 nodes\n",
"- define a `second_to_last` layer with 2 nodes\n",
"- define an output layer with 3 nodes\n",
"- create a model that connect input and output\n",
"- train it and make sure that it converges\n",
"- define a function between inputs and second_to_last layer\n",
"- recalculate the features and plot them"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.layers import Input\n",
"from tensorflow.keras.models import Model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"\n",
"inputs = Input(shape=(13,))\n",
"x = Dense(8, kernel_initializer='he_normal', activation='tanh')(inputs)\n",
"x = Dense(5, kernel_initializer='he_normal', activation='tanh')(x)\n",
"second_to_last = Dense(2, kernel_initializer='he_normal',\n",
" activation='tanh')(x)\n",
"outputs = Dense(3, activation='softmax')(second_to_last)\n",
"\n",
"model = Model(inputs=inputs, outputs=outputs)\n",
"\n",
"model.compile(RMSprop(learning_rate=0.05),\n",
" 'categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"\n",
"model.fit(Xsc, y_cat.values, batch_size=16, epochs=20, verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"features_function = K.function([inputs], [second_to_last])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"features = features_function([Xsc])[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.scatter(features[:, 0], features[:, 1], c=y)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 4 \n",
"\n",
"Keras offers the possibility to call a function at each epoch. These are Callbacks, and their [documentation is here](https://keras.io/callbacks/). Callbacks allow us to add some neat functionality. In this exercise we'll explore a few of them.\n",
"\n",
"- Split the data into train and test sets with a test_size = 0.3 and random_state=42\n",
"- Reset and recompile your model\n",
"- train the model on the train data using `validation_data=(X_test, y_test)`\n",
"- Use the `EarlyStopping` callback to stop your training if the `val_loss` doesn't improve\n",
"- Use the `ModelCheckpoint` callback to save the trained model to disk once training is finished\n",
"- Use the `TensorBoard` callback to output your training information to a `/tmp/` subdirectory\n",
"- Watch the next video for an overview of tensorboard"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"checkpointer = ModelCheckpoint(filepath=\"/tmp/udemy/weights.hdf5\",\n",
" verbose=1, save_best_only=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"earlystopper = EarlyStopping(monitor='val_loss', min_delta=0,\n",
" patience=1, verbose=1, mode='auto')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"tensorboard = TensorBoard(log_dir='/tmp/udemy/tensorboard/')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(Xsc, y_cat.values,\n",
" test_size=0.3,\n",
" random_state=42)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"\n",
"inputs = Input(shape=(13,))\n",
"\n",
"x = Dense(8, kernel_initializer='he_normal', activation='tanh')(inputs)\n",
"x = Dense(5, kernel_initializer='he_normal', activation='tanh')(x)\n",
"second_to_last = Dense(2, kernel_initializer='he_normal',\n",
" activation='tanh')(x)\n",
"outputs = Dense(3, activation='softmax')(second_to_last)\n",
"\n",
"model = Model(inputs=inputs, outputs=outputs)\n",
"\n",
"model.compile(RMSprop(learning_rate=0.05), 'categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"\n",
"model.fit(X_train, y_train, batch_size=32,\n",
" epochs=20, verbose=2,\n",
" validation_data=(X_test, y_test),\n",
" callbacks=[checkpointer, earlystopper, tensorboard])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Run Tensorboard with the command:\n",
"\n",
" tensorboard --logdir /tmp/udemy/tensorboard/\n",
" \n",
"and open your browser at http://localhost:6006"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

365
solutions/6 Convolutional Neural Networks Exercises Solution.ipynb Normal file
View File

@@ -0,0 +1,365 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Convolutional Neural Networks Exercises Solution"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"%matplotlib inline\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.utils import to_categorical"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten\n",
"import tensorflow.keras.backend as K"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"### Exercise 1\n",
"\n",
"You've been hired by a shipping company to overhaul the way they route mail, parcels and packages. They want to build an image recognition system capable of recognizing the digits in the zipcode on a package, so that it can be automatically routed to the correct location.\n",
"You are tasked to build the digit recognition system. Luckily, you can rely on the MNIST dataset for the intial training of your model!\n",
"\n",
"Build a deep convolutional neural network with at least two convolutional and two pooling layers before the fully connected layer.\n",
"\n",
"- Start from the network we have just built\n",
"- Insert a `Conv2D` layer after the first `MaxPool2D`, give it 64 filters.\n",
"- Insert a `MaxPool2D` after that one\n",
"- Insert an `Activation` layer\n",
"- retrain the model\n",
"- does performance improve?\n",
"- how many parameters does this new model have? More or less than the previous model? Why?\n",
"- how long did this second model take to train? Longer or shorter than the previous model? Why?\n",
"- did it perform better or worse than the previous model?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.datasets import mnist"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"(X_train, y_train), (X_test, y_test) = mnist.load_data(('/tmp/mnist.npz'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train = X_train.astype('float32') / 255.0\n",
"X_test = X_test.astype('float32') / 255.0\n",
"\n",
"X_train = X_train.reshape(-1, 28, 28, 1)\n",
"X_test = X_test.reshape(-1, 28, 28, 1)\n",
"\n",
"y_train_cat = to_categorical(y_train, 10)\n",
"y_test_cat = to_categorical(y_test, 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"\n",
"model = Sequential()\n",
"\n",
"model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
"model.add(MaxPool2D(pool_size=(2, 2)))\n",
"\n",
"model.add(Conv2D(64, (3, 3), activation='relu'))\n",
"model.add(MaxPool2D(pool_size=(2, 2)))\n",
"\n",
"model.add(Flatten())\n",
"\n",
"model.add(Dense(128, activation='relu'))\n",
"\n",
"model.add(Dense(10, activation='softmax'))\n",
"\n",
"model.compile(loss='categorical_crossentropy',\n",
" optimizer='rmsprop',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train, y_train_cat, batch_size=128,\n",
" epochs=2, verbose=1, validation_split=0.3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.evaluate(X_test, y_test_cat)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Exercise 2\n",
"\n",
"Pleased with your performance with the digits recognition task, your boss decides to challenge you with a harder task. Their online branch allows people to upload images to a website that generates and prints a postcard that is shipped to destination. Your boss would like to know what images people are loading on the site in order to provide targeted advertising on the same page, so he asks you to build an image recognition system capable of recognizing a few objects. Luckily for you, there's a dataset ready made with a collection of labeled images. This is the [Cifar 10 Dataset](http://www.cs.toronto.edu/~kriz/cifar.html), a very famous dataset that contains images for 10 different categories:\n",
"\n",
"- airplane \t\t\t\t\t\t\t\t\t\t\n",
"- automobile \t\t\t\t\t\t\t\t\t\t\n",
"- bird \t\t\t\t\t\t\t\t\t\t\n",
"- cat \t\t\t\t\t\t\t\t\t\t\n",
"- deer \t\t\t\t\t\t\t\t\t\t\n",
"- dog \t\t\t\t\t\t\t\t\t\t\n",
"- frog \t\t\t\t\t\t\t\t\t\t\n",
"- horse \t\t\t\t\t\t\t\t\t\t\n",
"- ship \t\t\t\t\t\t\t\t\t\t\n",
"- truck\n",
"\n",
"In this exercise we will reach the limit of what you can achieve on your laptop and get ready for the next session on cloud GPUs.\n",
"\n",
"Here's what you have to do:\n",
"- load the cifar10 dataset using `keras.datasets.cifar10.load_data()`\n",
"- display a few images, see how hard/easy it is for you to recognize an object with such low resolution\n",
"- check the shape of X_train, does it need reshape?\n",
"- check the scale of X_train, does it need rescaling?\n",
"- check the shape of y_train, does it need reshape?\n",
"- build a model with the following architecture, and choose the parameters and activation functions for each of the layers:\n",
" - conv2d\n",
" - conv2d\n",
" - maxpool\n",
" - conv2d\n",
" - conv2d\n",
" - maxpool\n",
" - flatten\n",
" - dense\n",
" - output\n",
"- compile the model and check the number of parameters\n",
"- attempt to train the model with the optimizer of your choice. How fast does training proceed?\n",
"- If training is too slow (as expected) stop the execution and move to the next session!"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.datasets import cifar10"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"(X_train, y_train), (X_test, y_test) = cifar10.load_data()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.imshow(X_train[1])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train = X_train.astype('float32') / 255.0\n",
"X_test = X_test.astype('float32') / 255.0"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_train.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_train_cat = to_categorical(y_train, 10)\n",
"y_test_cat = to_categorical(y_test, 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_train_cat.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
"model.add(Conv2D(32, (3, 3),\n",
" padding='same',\n",
" input_shape=(32, 32, 3),\n",
" activation='relu'))\n",
"model.add(Conv2D(32, (3, 3), activation='relu'))\n",
"model.add(MaxPool2D(pool_size=(2, 2)))\n",
"\n",
"model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))\n",
"model.add(Conv2D(64, (3, 3), activation='relu'))\n",
"model.add(MaxPool2D(pool_size=(2, 2)))\n",
"\n",
"model.add(Flatten())\n",
"model.add(Dense(512, activation='relu'))\n",
"model.add(Dense(10, activation='softmax'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss='categorical_crossentropy',\n",
" optimizer='rmsprop',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train, y_train_cat,\n",
" batch_size=32,\n",
" epochs=2,\n",
" validation_data=(X_test, y_test_cat),\n",
" shuffle=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

333
solutions/8 Recurrent Neural Networks Exercises Solutions.ipynb Normal file
View File

@@ -0,0 +1,333 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Recurrent Neural Networks"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"%matplotlib inline\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Time series forecasting"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pandas.tseries.offsets import MonthEnd\n",
"\n",
"df = pd.read_csv('../data/cansim-0800020-eng-6674700030567901031.csv',\n",
" skiprows=6, skipfooter=9,\n",
" engine='python')\n",
"\n",
"df['Adjustments'] = pd.to_datetime(df['Adjustments']) + MonthEnd(1)\n",
"df = df.set_index('Adjustments')\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"split_date = pd.Timestamp('01-01-2011')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train = df.loc[:split_date, ['Unadjusted']]\n",
"test = df.loc[split_date:, ['Unadjusted']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import MinMaxScaler\n",
"\n",
"sc = MinMaxScaler()\n",
"\n",
"train_sc = sc.fit_transform(train)\n",
"test_sc = sc.transform(test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train_sc_df = pd.DataFrame(train_sc, columns=['Scaled'], index=train.index)\n",
"test_sc_df = pd.DataFrame(test_sc, columns=['Scaled'], index=test.index)\n",
"\n",
"for s in range(1, 13):\n",
" train_sc_df['shift_{}'.format(s)] = train_sc_df['Scaled'].shift(s)\n",
" test_sc_df['shift_{}'.format(s)] = test_sc_df['Scaled'].shift(s)\n",
"\n",
"X_train = train_sc_df.dropna().drop('Scaled', axis=1)\n",
"y_train = train_sc_df.dropna()[['Scaled']]\n",
"\n",
"X_test = test_sc_df.dropna().drop('Scaled', axis=1)\n",
"y_test = test_sc_df.dropna()[['Scaled']]\n",
"\n",
"X_train = X_train.values\n",
"X_test= X_test.values\n",
"\n",
"y_train = y_train.values\n",
"y_test = y_test.values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 1\n",
"\n",
"In the model above we reshaped the input shape to: `(num_samples, 1, 12)`, i.e. we treated a window of 12 months as a vector of 12 coordinates that we simultaneously passed to all the LSTM nodes. An alternative way to look at the problem is to reshape the input to `(num_samples, 12, 1)`. This means we consider each input window as a sequence of 12 values that we will pass in sequence to the LSTM. In principle this looks like a more accurate description of our situation. But does it yield better predictions? Let's check it.\n",
"\n",
"- Reshape `X_train` and `X_test` so that they represent a set of univariate sequences\n",
"- retrain the same LSTM(6) model, you'll have to adapt the `input_shape`\n",
"- check the performance of this new model, is it better at predicting the test data?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train_t = X_train.reshape(X_train.shape[0], 12, 1)\n",
"X_test_t = X_test.reshape(X_test.shape[0], 12, 1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train_t.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import LSTM, Dense\n",
"import tensorflow.keras.backend as K\n",
"from tensorflow.keras.callbacks import EarlyStopping"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"model = Sequential()\n",
"\n",
"model.add(LSTM(6, input_shape=(12, 1)))\n",
"\n",
"model.add(Dense(1))\n",
"\n",
"model.compile(loss='mean_squared_error', optimizer='adam')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"early_stop = EarlyStopping(monitor='loss', patience=1, verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train_t, y_train, epochs=600,\n",
" batch_size=32, verbose=0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_pred = model.predict(X_test_t)\n",
"plt.plot(y_test)\n",
"plt.plot(y_pred)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"## Exercise 2\n",
"\n",
"RNN models can be applied to images too. In general we can apply them to any data where there's a connnection between nearby units. Let's see how we can easily build a model that works with images.\n",
"\n",
"- Load the MNIST data, by now you should be able to do it blindfolded :)\n",
"- reshape it so that an image looks like a long sequence of pixels\n",
"- create a recurrent model and train it on the training data\n",
"- how does it perform compared to a fully connected? How does it compare to Convolutional Neural Networks?\n",
"\n",
"(feel free to run this exercise on a cloud GPU if it's too slow on your laptop)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.datasets import mnist\n",
"from tensorflow.keras.utils import to_categorical"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
"X_train = X_train.astype('float32') / 255.0\n",
"X_test = X_test.astype('float32') / 255.0\n",
"y_train_cat = to_categorical(y_train, 10)\n",
"y_test_cat = to_categorical(y_test, 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train = X_train.reshape(X_train.shape[0], -1, 1)\n",
"X_test = X_test.reshape(X_test.shape[0], -1, 1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(X_train.shape)\n",
"print(X_test.shape)\n",
"print(y_train_cat.shape)\n",
"print(y_test_cat.shape)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# define the model\n",
"K.clear_session()\n",
"model = Sequential()\n",
"model.add(LSTM(32, input_shape=X_train.shape[1:]))\n",
"model.add(Dense(10, activation='softmax'))\n",
"\n",
"# compile the model\n",
"model.compile(loss='categorical_crossentropy',\n",
" optimizer='rmsprop',\n",
" metrics=['accuracy'])\n",
"\n",
"model.fit(X_train, y_train_cat,\n",
" batch_size=32,\n",
" epochs=100,\n",
" validation_split=0.3,\n",
" shuffle=True,\n",
" verbose=2,\n",
" )\n",
"\n",
"model.evaluate(X_test, y_test_cat)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

520
solutions/9 Improving performance Exercises Solutions.ipynb Normal file
View File

@@ -0,0 +1,520 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 9 Improving performance Exercises Solutions"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"%matplotlib inline\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 1\n",
"\n",
"- Reload the IMDB data keeping only the first 20000 most common words\n",
"- pad the reviews to a shorter length (eg. 70 or 80), this time make sure you keep the first part of the review if it's longer than the maximum length\n",
"- re run the model (remember to set max_features correctly)\n",
"- does it train faster this time?\n",
"- do you get a better performance?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.datasets import imdb\n",
"from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Embedding, LSTM, Dense"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"max_features = 20000\n",
"skip_top = 200"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"(X_train, y_train), (X_test, y_test) = imdb.load_data('/tmp/imdb.npz',\n",
" num_words=max_features,\n",
" start_char=1,\n",
" oov_char=2,\n",
" index_from=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"maxlen = 80"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train_pad = pad_sequences(X_train, maxlen=maxlen, truncating='post')\n",
"X_test_pad = pad_sequences(X_test, maxlen=maxlen, truncating='post')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
"model.add(Embedding(max_features, 128))\n",
"model.add(LSTM(64, dropout=0.2, recurrent_dropout=0.2))\n",
"model.add(Dense(1, activation='sigmoid'))\n",
"\n",
"model.compile(loss='binary_crossentropy',\n",
" optimizer='adam',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_train[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(X_train_pad, y_train,\n",
" batch_size=32,\n",
" epochs=2,\n",
" validation_split=0.3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"score, acc = model.evaluate(X_test_pad, y_test)\n",
"print('Test score:', score)\n",
"print('Test accuracy:', acc)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 2\n",
"\n",
"- Reload the digits data as above\n",
"- define a function repeated_training_reg_dropout that adds regularization and dropout to a fully connected network\n",
"- compare the performance with/witouth dropout and regularization like we did for batch normalization\n",
"- do you get a better performance?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.datasets import load_digits\n",
"from tensorflow.keras.utils import to_categorical\n",
"from sklearn.model_selection import train_test_split\n",
"from tensorflow.keras.layers import Dropout\n",
"import tensorflow.keras.backend as K"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"digits = load_digits()\n",
"X, y = digits.data, digits.target\n",
"y_cat = to_categorical(y)\n",
"\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y_cat, test_size=0.3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def repeated_training_reg_dropout(X_train,\n",
" y_train,\n",
" X_test,\n",
" y_test,\n",
" units=512,\n",
" activation='sigmoid',\n",
" optimizer='sgd',\n",
" do_dropout=False,\n",
" rate=0.3,\n",
" kernel_regularizer='l2',\n",
" epochs=10,\n",
" repeats=3):\n",
" histories = []\n",
" \n",
" for repeat in range(repeats):\n",
" K.clear_session()\n",
"\n",
" model = Sequential()\n",
" \n",
" # first fully connected layer\n",
" model.add(Dense(units,\n",
" input_shape=X_train.shape[1:],\n",
" kernel_initializer='normal',\n",
" kernel_regularizer=kernel_regularizer,\n",
" activation=activation))\n",
" if do_dropout:\n",
" model.add(Dropout(rate))\n",
"\n",
" # second fully connected layer\n",
" model.add(Dense(units,\n",
" kernel_initializer='normal',\n",
" kernel_regularizer=kernel_regularizer,\n",
" activation=activation))\n",
" if do_dropout:\n",
" model.add(Dropout(rate))\n",
"\n",
" # third fully connected layer\n",
" model.add(Dense(units,\n",
" kernel_initializer='normal',\n",
" kernel_regularizer=kernel_regularizer,\n",
" activation=activation))\n",
" if do_dropout:\n",
" model.add(Dropout(rate))\n",
"\n",
" # output layer\n",
" model.add(Dense(10, activation='softmax'))\n",
" \n",
" model.compile(optimizer,\n",
" 'categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
"\n",
" h = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, verbose=0)\n",
" histories.append([h.history['accuracy'], h.history['val_accuracy']])\n",
" print(repeat, end=' ')\n",
"\n",
" histories = np.array(histories)\n",
" \n",
" # calculate mean and standard deviation across repeats:\n",
" mean_acc = histories.mean(axis=0)\n",
" std_acc = histories.std(axis=0)\n",
" print()\n",
" \n",
" return mean_acc[0], std_acc[0], mean_acc[1], std_acc[1]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mean_acc, std_acc, mean_acc_val, std_acc_val = repeated_training_reg_dropout(X_train,\n",
" y_train,\n",
" X_test,\n",
" y_test,\n",
" do_dropout=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mean_acc_do, std_acc_do, mean_acc_val_do, std_acc_val_do = repeated_training_reg_dropout(X_train,\n",
" y_train,\n",
" X_test,\n",
" y_test,\n",
" do_dropout=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def plot_mean_std(m, s):\n",
" plt.plot(m)\n",
" plt.fill_between(range(len(m)), m-s, m+s, alpha=0.1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plot_mean_std(mean_acc, std_acc)\n",
"plot_mean_std(mean_acc_val, std_acc_val)\n",
"plot_mean_std(mean_acc_do, std_acc_do)\n",
"plot_mean_std(mean_acc_val_do, std_acc_val_do)\n",
"plt.ylim(0, 1.01)\n",
"plt.title(\"Dropout and Regularization Accuracy\")\n",
"plt.xlabel('Epochs')\n",
"plt.ylabel('Accuracy')\n",
"plt.legend(['Train', 'Test', 'Train with Dropout and Regularization', 'Test with Dropout and Regularization'], loc='best')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 3\n",
"\n",
"This is a very long and complex exercise, that should give you an idea of a real world scenario. Feel free to look at the solution if you feel lost. Also, feel free to run this with a GPU, in which case you don't need to download the data.\n",
"\n",
"If you are running this locally, download and unpack the male/female pictures from [here](https://www.dropbox.com/s/nov493om2jmh2gp/male_female.tgz?dl=0). These images and labels were obtained from [Crowdflower](https://www.crowdflower.com/data-for-everyone/).\n",
"\n",
"Your goal is to build an image classifier that will recognize the gender of a person from pictures.\n",
"\n",
"- Have a look at the directory structure and inspect a couple of pictures\n",
"- Design a model that will take a color image of size 64x64 as input and return a binary output (female=0/male=1)\n",
"- Feel free to introduce any regularization technique in your model (Dropout, Batch Normalization, Weight Regularization)\n",
"- Compile your model with an optimizer of your choice\n",
"- Using `ImageDataGenerator`, define a train generator that will augment your images with some geometric transformations. Feel free to choose the parameters that make sense to you.\n",
"- Define also a test generator, whose only purpose is to rescale the pixels by 1./255\n",
"- use the function `flow_from_directory` to generate batches from the train and test folders. Make sure you set the `target_size` to 64x64.\n",
"- Use the `model.fit_generator` function to fit the model on the batches generated from the ImageDataGenerator. Since you are streaming and augmenting the data in real time you will have to decide how many batches make an epoch and how many epochs you want to run\n",
"- Train your model (you should get to at least 85% accuracy)\n",
"- Once you are satisfied with your training, check a few of the misclassified pictures. Are those sensible errors?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# If you are running this locally\n",
"# uncomment the next 4 lines to download, extract and set the data path:\n",
"# !wget 'https://www.dropbox.com/s/nov493om2jmh2gp/male_female.tgz?dl=1' -O ../data/male_female.tgz\n",
"# data_path = '../data/male_female'\n",
"# !mkdir -p {data_path}\n",
"# !tar -xzvf ../data/male_female.tgz --directory {data_path}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tensorflow.keras.layers import Conv2D\n",
"from tensorflow.keras.layers import MaxPooling2D\n",
"from tensorflow.keras.layers import Flatten\n",
"from tensorflow.keras.layers import BatchNormalization\n",
"from itertools import islice\n",
"from tensorflow.keras.preprocessing.image import ImageDataGenerator"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"K.clear_session()\n",
"\n",
"model = Sequential()\n",
"model.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))\n",
"model.add(MaxPooling2D(pool_size = (2, 2)))\n",
"model.add(BatchNormalization())\n",
"\n",
"model.add(Conv2D(64, (3, 3), activation = 'relu'))\n",
"model.add(MaxPooling2D(pool_size = (2, 2)))\n",
"model.add(BatchNormalization())\n",
"\n",
"model.add(Conv2D(64, (3, 3), activation = 'relu'))\n",
"model.add(MaxPooling2D(pool_size = (2, 2)))\n",
"model.add(BatchNormalization())\n",
"\n",
"model.add(Flatten())\n",
"\n",
"model.add(Dense(128, activation = 'relu'))\n",
"model.add(Dense(1, activation = 'sigmoid'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer = 'adam',\n",
" loss = 'binary_crossentropy',\n",
" metrics = ['accuracy'])\n",
"\n",
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train_gen = ImageDataGenerator(rescale = 1./255,\n",
" width_shift_range=0.1,\n",
" height_shift_range=0.1,\n",
" rotation_range = 10,\n",
" shear_range = 0.2,\n",
" zoom_range = 0.2,\n",
" horizontal_flip = True)\n",
"\n",
"test_gen = ImageDataGenerator(rescale = 1./255)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train = train_gen.flow_from_directory(data_path + '/train',\n",
" target_size = (64, 64),\n",
" batch_size = 16,\n",
" class_mode = 'binary')\n",
"\n",
"test = test_gen.flow_from_directory(data_path + '/test',\n",
" target_size = (64, 64),\n",
" batch_size = 16,\n",
" class_mode = 'binary')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.fit(train,\n",
" steps_per_epoch = 800,\n",
" epochs = 200,\n",
" validation_data = test,\n",
" validation_steps = 200)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X_test = []\n",
"y_test = []\n",
"for ts in islice(test, 50):\n",
" X_test.append(ts[0])\n",
" y_test.append(ts[1])\n",
"\n",
"X_test = np.concatenate(X_test)\n",
"y_test = np.concatenate(y_test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_pred = model.predict_classes(X_test).ravel()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"np.argwhere(y_test != y_pred).ravel()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.imshow(X_test[14])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 2
}