Lab1-LR
ipynb13 kBPython · 32 cellsSolution of TP 1: Linear regression
TP1: Linear Regression with Gradient Descent
import numpy as np
import matplotlib.pyplot as plt
import pandas as pdPart A- Univariate Linear Regression
Step 1- Data Preparation
We start with a simple dataset relating house size to price. The goal is to fit a line to predict price from size.
# Generate a simple Dataset (house size vs. price)
x = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) # house size in 100 m²
y = np.array([3, 4, 2, 5, 6, 7, 8, 9, 10]) # house price in $100k
plt.scatter(x, y, color="blue", marker="o")
plt.xlabel("House Size (100 m²)")
plt.ylabel("House Price ($100k)")
plt.title("Training Data: House Size vs. Price")
plt.show()Step 2- Hypothesis Function
Defines the model: a straight line with parameters (intercept) and (slope).
def hypothesis(x, theta0, theta1):
return theta0 + theta1 * xStep 3- Cost Function (Mean Squared Error)
Measures how well the model fits the data. Lower cost means better fit.
def computeCost(x, y, theta0, theta1):
m = len(y)
predictions = hypothesis(x, theta0, theta1)
cost = (1 / (2 * m)) * np.sum((predictions - y) ** 2)
return costStep 4- Gradient Descent Update Rule
Algorithm to minimize the cost function by updating and iteratively.
def gradient_descent(x, y, theta0, theta1, alpha, iterations):
m = len(y)
cost_history = []
for _ in range(iterations):
predictions = hypothesis(x, theta0, theta1)
error = predictions - y
# Compute gradient using all data points
theta0 -= alpha * (1 / m) * np.sum(error)
theta1 -= alpha * (1 / m) * np.sum(error * x)
# Track cost after each iteration
cost_history.append(computeCost(x, y, theta0, theta1))
return theta0, theta1, cost_historyStep 5- Train the Model
Run gradient descent to find the best parameters for the linear model.
theta0 = 0.0
theta1 = 0.0
alpha = 0.01
iterations = 1000
theta0, theta1, cost_history = gradient_descent(x, y, theta0, theta1, alpha, iterations)
print("Final theta0:", theta0)
print("Final theta1:", theta1)
# Plot cost convergence
plt.plot(cost_history)
plt.xlabel("Iterations")
plt.ylabel("Cost J(θ)")
plt.title("Cost Function Convergence")
plt.show()Step 6- Results & Visualization
Visualize the fitted line and check how well it matches the data.
# Plot Regression Line
plt.scatter(x, y, color="blue", marker="o", label="Training Data")
plt.plot(x, hypothesis(x, theta0, theta1), color="red", label="Regression Line")
plt.xlabel("House Size (100 m²)")
plt.ylabel("House Price ($100k)")
plt.title("Linear Regression Fit")
plt.legend()
plt.show()Part B- Multivariate Linear Regression
Now we use more than one feature (e.g., size and number of rooms) to predict price.
# Example dataset (size, rooms -> price)
data = {
"size": [2100, 1600, 2400, 1416, 3000],
"rooms": [3, 2, 4, 2, 4],
"price": [400, 330, 369, 232, 540],
}
df = pd.DataFrame(data)
x = df[["size", "rooms"]].values
y = df["price"].values
# Feature Normalization
x = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
# Add intercept column
x = np.c_[np.ones(x.shape[0]), x]
print("X shape:", x.shape)
print("y shape:", y.shape)Step 2- Hypothesis (Vectorized)
The model predicts using a vector of parameters and features: .
def hypothesis(x, theta):
return x.dot(theta)Step 3- Cost Function (Vectorized)
Measures the fit for all features and parameters together.
def compute_cost(x, y, theta):
m = len(y)
predictions = hypothesis(x, theta)
error = predictions - y
cost = (1 / (2 * m)) * np.dot(error.T, error)
return costStep 4- Gradient Descent Update Rule (Vectorized)
Update all parameters at once using the gradient of the cost function.
def gradient_descent(x, y, theta, alpha, iterations):
m = len(y)
cost_history = []
for _ in range(iterations):
predictions = hypothesis(x, theta)
error = predictions - y
gradient = (1 / m) * np.dot(x.T, error)
theta = theta - alpha * gradient
cost_history.append(compute_cost(x, y, theta))
return theta, cost_historyStep 5- Train and Compare
Train the multivariate model and observe cost reduction over iterations.
theta = np.zeros(x.shape[1]) # Initialize theta
alpha = 0.01
iterations = 1000
theta, cost_history = gradient_descent(x, y, theta, alpha, iterations)
print("Learned Parameters:", theta)
plt.plot(cost_history)
plt.xlabel("Iterations")
plt.ylabel("Cost J(θ)")
plt.title("MultiVariate Cost Convergence")Step 6- Compare with Scikit-Learn
Compare your implementation with a standard library to check correctness.
note: the values would not look close because we did not normalize the data for the Scikit-learn model
from sklearn.linear_model import LinearRegression
model = LinearRegression()
model.fit(df[["size", "rooms"]], df["price"])
print("Scikit-Learn Coefficients:", model.intercept_, model.coef_)Part C- Polynomial Regression
Now we extend linear regression to polynomial regression to capture non-linear relationships. We'll use a dataset with features: size, number of bedrooms, number of floors, and age to predict house price.
# Example dataset (size, bedrooms, floors, age -> price)
data_poly = {
"size": [2104, 1416, 1534, 852],
"bedrooms": [5, 3, 3, 2],
"floors": [1, 2, 2, 1],
"age": [45, 40, 30, 36],
"price": [460, 232, 315, 178],
}
df_poly = pd.DataFrame(data_poly)
X_poly = df_poly[["size", "bedrooms", "floors", "age"]].values
y_poly = df_poly["price"].values# Create polynomial features (degree 2)
from sklearn.preprocessing import PolynomialFeatures
poly = PolynomialFeatures(degree=2, include_bias=False)
X_poly_expanded = poly.fit_transform(X_poly)
print("Original shape:", X_poly.shape)
print("Expanded shape:", X_poly_expanded.shape)# Fit polynomial regression model
model_poly = LinearRegression()
model_poly.fit(X_poly_expanded, y_poly)
print("Polynomial Regression Coefficients:")
print(model_poly.intercept_, model_poly.coef_)Visualization and Prediction Example
You can visualize the fit for one feature (e.g., size) or show predictions for new data points.
# Predict price for a new house (example)
new_house = np.array([[2000, 3, 2, 10]])
new_house_poly = poly.transform(new_house)
predicted_price = model_poly.predict(new_house_poly)
print(f"Predicted price for house {new_house[0]}: ${predicted_price[0]:.2f}k")