数据分析bit

import json
from collections import defaultdict
from collections import Counter
from pandas import DataFrame, Series
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

path = ‘example.txt’

with open(path) as f:

for line in f.readline():

record = json.load(str(line))

print(record)

teststr = ‘{“user”: { “user_id”: 2131, “name”: “John”, “gender”: 0, “thumb_url”: “sd”, “money”: 23, “cash”: 2, “material”: 5}}’

cr = json.loads(teststr)

print(cr)

将字符串转为字典

records = [json.loads(line) for line in open(path)]

print(records[0][‘tz’])

time_zones = [rec[‘tz’] for rec in records if ‘tz’ in rec]

print(time_zones)

def get_counts(squence):
counts = {}
for x in squence:
if x in counts:
counts[x] += 1
else:
counts[x] = 1
return counts

count = get_counts(time_zones)

print(count)

print(len(time_zones))

counts = Counter(time_zones)

print(counts)

def top_counts(count_dict, n=10):
values_key_pairs = [(countes, tz) for tz, countes in count_dict.items()]
values_key_pairs.sort()
return values_key_pairs[-n:]

print(top_counts(count))

用pandas计数

frame = DataFrame(records)

tz_count = frame[‘tz’].value_counts()

print(frame[‘tz’][:10])

print(tz_count)

用绘图库生成图片

fillna函数可以替换缺失值NA

clean_tz = frame[‘tz’].fillna(‘Missing’)
clean_tz[clean_tz == ‘’] = ‘unkown’
tz_counts = clean_tz.value_counts()

print(tz_counts[:10])

x = tz_counts[:10]
x.plot(kind=‘barh’, rot=0)

plt.show()

value_counts()是Series中的方法

信息分解

print(frame[:1])
results = Series([y.split()[0] for y in frame.a.dropna()])

print(results[:5])

cframe = (frame[frame.a.notnull()])

print(cframe)

operating_system = np.where(cframe[‘a’].str.contains(‘Windows’), ‘Windows’
, ‘Not Windows’)
print(operating_system[:5])

根据时区和系统多数据分组

by_tz_os = cframe.groupby([‘tz’, operating_system])
agg_counts = by_tz_os.size().unstack().fillna(0)

print(agg_counts[:10])

print(by_tz_os)

用于按升序排列

indexer = agg_counts.sum(1).argsort()

print(indexer)

count_subset = agg_counts.take(indexer)[-10:]
count_subset.plot(kind=‘barh’, stacked=True)

各行规范化总计为1

normed_subset = count_subset.div(count_subset.sum(1), axis=0)
normed_subset.plot(kind=‘barh’, stacked=True)

plt.show()