一、数据收集
数据收集通过 Twitter API，搜集 US 境内全部 Twitter 数据，以 JSON 格式存储在 txt 文件中。
二、数据读取
从 txt 文件中，以 JSON 格式去获取每条 tweet 的信息，然后存储于 csv 文件中。读取时候的编码选的是 gbk。
代码如下：
from math import radians, sin
import json, os, codecs
# area of bounding box
def area(lon1, lat1, lon2, lat2):
lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
r = 6372
return abs(r**2 * (lon2 - lon1) * (sin(lat2) - sin(lat1)))
# tweets of txt to csv
def txt2csv(foldername, filename):
files = os.listdir(foldername)
os.chdir(foldername)
fo = open(filename, "w")
# fo.write("\ufeff")
fo.write("id,created_at,coordinates,co_lon,co_lat,geo,geo_lat,geo_lon," +
"user_location,place_type,place_name," +
"place_full_name,place_country,place_bounding_box,pb_avg_lon,pb_avg_lat," +
"min_lon,min_lat,max_lon,max_lat,bb_area,lang,source,text")
count = 0
for file in files:
# determine is file or directory
if os.path.isdir(file):
continue
count += 1
print(count, ":", file)
tweets_file = open(file, "r")
for line in tweets_file:
try:
tweet = json.loads(line)
csv_text = "\n"
# id
csv_text += tweet["id_str"]
csv_text += ","
# created_at
csv_text += str(tweet["created_at"])
csv_text += ","
# coordinates
if (tweet["coordinates"]):
csv_text += "Yes,"
csv_text += str(tweet["coordinates"]["coordinates"][0])
csv_text += ","
csv_text += str(tweet["coordinates"]["coordinates"][1])
else:
csv_text += "None,None,None"
csv_text += ","
# geo
if (tweet["geo"]):
csv_text += "Yes,"
csv_text += str(tweet["geo"]["coordinates"][0])
csv_text += ","
csv_text += str(tweet["geo"]["coordinates"][1])
else:
csv_text += "None,None,None"
csv_text += ","
# user->location
ul = str(tweet["user"]["location"])
ul = ul.replace("\n", " ")
ul = ul.replace("\"", "")
ul = ul.replace("\'", "")
csv_text += "\"" + ul + "\""
csv_text += ","
# place->type
csv_text += str(tweet["place"]["place_type"])
csv_text += ","
# place->name
csv_text += "\"" + str(tweet["place"]["name"]) + "\""
csv_text += ","
# place->full_name
csv_text += "\"" + str(tweet["place"]["full_name"]) + "\""
csv_text += ","
# place->country
csv_text += "\"" + str(tweet["place"]["country"]) + "\""
csv_text += ","
# place->bounding_box
if (tweet["place"]["bounding_box"]["coordinates"]):
# min_lon
min_lon = tweet["place"]["bounding_box"]["coordinates"][0][0][0]
# min_lat
min_lat = tweet["place"]["bounding_box"]["coordinates"][0][0][1]
# max_lon
max_lon = tweet["place"]["bounding_box"]["coordinates"][0][2][0]
# max_lat
max_lat = tweet["place"]["bounding_box"]["coordinates"][0][2][1]
# avg of lon and lat
lon = (min_lon + max_lon)/2
lat = (min_lat + max_lat)/2
# area of bounding box
area_bb = area(min_lon, min_lat, max_lon, max_lat)
csv_text += "Yes,"
csv_text += str(lon)
csv_text += ","
csv_text += str(lat)
csv_text += ","
csv_text += str(min_lon)
csv_text += ","
csv_text += str(min_lat)
csv_text += ","
csv_text += str(max_lon)
csv_text += ","
csv_text += str(max_lat)
csv_text += ","
csv_text += str(area_bb)
else:
csv_text += "None, None, None"
csv_text += ","
# lang
csv_text += str(tweet["lang"])
csv_text += ","
# source
csv_text += "\"" + str(tweet["source"]) + "\""
csv_text += ","
# text
# replace carriage return, double quotation marks, single quotation marks with space or nothing
text = str(tweet["text"])
text = text.replace("\r", " ")
text = text.replace("\n", " ")
text = text.replace("\"", "")
text = text.replace("\'", "")
csv_text += "\"" + text + "\""
fo.write(csv_text)
except:
continue
fo.close()
txt2csv(r"E:\USA\test", r"D:\OneDrive - UNSW\01-UNSW\02-Papers_Plan\02-CCIS\04-US_Tweets\tt.csv")
import pandas as pd
df = pd.read_csv(r"D:\OneDrive - UNSW\01-UNSW\02-Papers_Plan\02-CCIS\04-US_Tweets\tt.csv", encoding='gbk')
df.head()
数据的显示效果如下：
一共是 24 列，分别存储与时间和地点相关的信息，包括创建时间、经纬度、text 信息等。
三、数据处理
3.1 获取 tweets 总数量
实现起来很简单，还要计算出有多少列就行。
代码如下：
import pandas as pd df = pd.read_csv(r"D:\OneDrive - UNSW\01-UNSW\02-Papers_Plan\02-CCIS\04-US_Tweets\tt.csv", encoding='gbk') # 数据量 df.shape
结果类似 (715, 24)，说明有 715 条记录。
3.2 获取不重复 tweets 总数量
由于在收集的过程中可能重复提取，因此需要进行删除重复数据
代码如下：
# delete duplicate tweets df = df.drop_duplicates(['id']) # 无重复数据量 df.shape
显示结果如上
3.3 修改某些列的数据类型
默认的很多列都是 object 类型，为了进行计算需要进行修改，例如时间的列修改成 datetime 类型，经纬度为 float 等。
代码如下：
# change data type to datetime
# co_lon and co_lat are NONE sometimes
df = df.astype({"created_at":"datetime64[ns]"})
修改之后可以提取其中的年与日信息。
3.4 获取 tweets 的来源
主要是查询是 web 还是 iPhone、Android、Instagram 等。
代码如下：
# get total number of every source
print("\nsource counts:\n\n", df.source.value_counts())
会将不同来源的数量按大到小打印出来。
3.5 获取 geo-tagged tweets 数量
获取带有地理信息的 tweets 数量。
代码如下：
# get total number of tweets with goe-tags
print("\nGeotagged tweets counts:\n\n", df.coordinates.value_counts())
3.6 获取位于 US 境内并且为 ENG 的 tweets 数量
代码如下：
# get tweets from Aus
df = df[df['place_country'] == 'United States']
# get English tweets
df = df[df['lang'] == 'en']
df.shape
print("\n US English tweets count: ", df.shape[0])