import random
import numpy as np
import scipy as sp
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
import statsmodels.formula.api as smf
sns.set_context("talk")
# Anscombe’s quartet Anscombe’s quartet comprises of four datasets, and is rather famous. Why? You’ll find out in this exercise.
anascombe = pd.read_csv('data/anscombe.csv')
anascombe.head()
.dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; }
dataset | x | y | |
---|---|---|---|
0 | I | 10.0 | 8.04 |
1 | I | 8.0 | 6.95 |
2 | I | 13.0 | 7.58 |
3 | I | 9.0 | 8.81 |
4 | I | 11.0 | 8.33 |
Part 1
For each of the four datasets…
- Compute the mean and variance of both x and y
- Compute the correlation coefficient between x and y
- Compute the linear regression line:
(hint: use statsmodels and look at the Statsmodels notebook)
def get_data(data):
return pd.Series([data['x'].mean(), data['x'].var(), data['y'].mean(), data['y'].var()],index=['x均值', 'x方差', 'y均值', 'y方差'])
dataset_name = anascombe.dataset.unique()
group = anascombe.groupby(by=list(["dataset"]))
for name in dataset_name:
data = group.get_group(name)
print('dataset: ', name)
print(pd.DataFrame(get_data(data)))
print('系数')
print(data.corr(),'\n')
x = data['x']
X = sm.add_constant(data['x'])
y = data['y']
model = sm.OLS(y,X)
results = model.fit()
print(results.params)
y_fitted = results.fittedvalues
fig, ax = plt.subplots()
ax.plot(x, y, 'o', label='data')
ax.plot(x, y_fitted, 'r-',label='OLS')
ax.legend(loc='best')
plt.show()
print('\n')
dataset: I
0
x均值 9.000000
x方差 11.000000
y均值 7.500909
y方差 4.127269
系数
x y
x 1.000000 0.816421
y 0.816421 1.000000
const 3.000091
x 0.500091
dtype: float64
dataset: II
0
x均值 9.000000
x方差 11.000000
y均值 7.500909
y方差 4.127629
系数
x y
x 1.000000 0.816237
y 0.816237 1.000000
const 3.000909
x 0.500000
dtype: float64
dataset: III
0
x均值 9.00000
x方差 11.00000
y均值 7.50000
y方差 4.12262
系数
x y
x 1.000000 0.816287
y 0.816287 1.000000
const 3.002455
x 0.499727
dtype: float64
dataset: IV
0
x均值 9.000000
x方差 11.000000
y均值 7.500909
y方差 4.123249
系数
x y
x 1.000000 0.816521
y 0.816521 1.000000
const 3.001727
x 0.499909
dtype: float64
Part 2
Using Seaborn, visualize all four datasets.
hint: use sns.FacetGrid combined with plt.scatter
g = sns.FacetGrid(anascombe, col="dataset", size=4)
g = g.map(plt.scatter, "x", "y", edgecolor="w")