A brief introduction to the popular Python libraries numpy and Pandas

numpy.ndarrayIs the main data structure in the NumPy library, it is a multi-dimensional array used to store and manipulate numerical data. NumPy is a powerful library for numerical calculations in Python. It numpy.ndarrayis its core data type and provides efficient numerical operations and a wide range of mathematical functions.

Here are numpy.ndarraysome of the important features and capabilities of :

1. Multidimensional array : numpy.ndarrayIt can be one-dimensional, two-dimensional or multi-dimensional array, which makes it very suitable for processing various complex data.

2. Data type : numpy.ndarrayCan contain elements of different data types, such as integers, floating point numbers, Boolean values, etc. Each array has a fixed data type, available through dtypethe property.

3. Broadcasting : NumPy supports broadcasting, which means that element-level operations can be performed on arrays of different shapes without the need for explicit loops.

4. Vectorized operations : NumPy provides a variety of mathematical, logical, and statistical functions that can be efficiently applied to numpy.ndarrayelements in without having to write loops.

5. Slicing and indexing : You can use slicing and indexing operations numpy.ndarrayto select and operate on subsets of data.

6. Mathematical operations : NumPy supports various mathematical operations, including addition, subtraction, multiplication, division, matrix operations, etc.

7. Random number generation : NumPy includes random number generation functions for generating random numbers or random arrays.

8. Data statistics : NumPy provides various statistical functions for calculating mean, variance, standard deviation, maximum, minimum, etc.

9. Linear algebra : NumPy includes linear algebra functions for matrix operations, solving systems of linear equations, and more.

10. File IO : You can use NumPy to read and write data to disk, supporting multiple file formats.

Here is a simple example showing how to create and operate numpy.ndarray:

import numpy as np

# 生成0到1之间的随机数，指定形状为(2, 3)
random_nums = np.random.rand(2, 3)
print("Random numbers from uniform distribution:")
print(random_nums)

# 生成标准正态分布的随机数，指定形状为(3, 4)
random_normal = np.random.randn(3, 4)
print("\nRandom numbers from standard normal distribution:")
print(random_normal)

# 生成整数随机数，范围在5到15之间，指定形状为(2, 2)
random_integers = np.random.randint(5, 15, size=(2, 2))
print("\nRandom integers between 5 and 15:")
print(random_integers)

# 生成均匀分布的随机数，范围在10到20之间，指定形状为(3,)
random_uniform = np.random.uniform(10, 20, size=3)
print("\nRandom numbers from uniform distribution:")
print(random_uniform)

# 生成符合正态分布的随机数，均值为2，标准差为3，指定形状为(14,)
random_normal_custom = np.random.normal(loc=2, scale=3, size=14)
print("\nRandom numbers from custom normal distribution:")
print(random_normal_custom)

# 从数组中随机选择元素，指定选择5次，允许重复选择，并指定概率分布
choices = ['apple', 'banana', 'cherry', 'date']
random_choices = np.random.choice(choices, size=5, replace=True, p=[0.2, 0.3, 0.4, 0.1])
print("\nRandom choices:")
print(random_choices)

# 随机打乱数组的元素顺序
arr = np.array([1, 2, 3, 4, 5])
np.random.shuffle(arr)
print("\nShuffled array:")
print(arr)

import numpy as np

# 创建一个包含整数的等差数列
arr = np.arange(start, stop, step)

# 参数说明：
# - start: 数列的起始值（包含在数列中）
# - stop: 数列的结束值（不包含在数列中）
# - step: 数列的步长（可选，默认为1）

# 示例 1: 创建一个包含0到9的整数数列
arr1 = np.arange(10)
# 结果: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

# 示例 2: 创建一个包含2到8（不包含8）的整数数列，步长为2
arr2 = np.arange(2, 8, 2)
# 结果: [2, 4, 6]

# 示例 3: 创建一个包含0到1的浮点数数列，步长为0.1
arr3 = np.arange(0, 1, 0.1)
# 结果: [0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]

import numpy as np

arr = np.array([1, 2, 3, 4, 5])
mean = np.mean(arr)  # 计算平均值
max_val = np.max(arr)  # 找到最大值

import numpy as np

np.random.seed(42)  # 设置种子为42
random_numbers = np.random.rand(5)  # 生成5个随机数

print(random_numbers)

random_int = np.random.randint(1, 100, 5)  # 生成1到99之间的5个整数随机数

random_uniform = np.random.rand(5)  # 生成包含5个随机数的数组

import numpy as np

# 创建一个一维数组
arr1 = np.array([1, 2, 3, 4, 5])

# 创建一个二维数组
arr2 = np.array([[1, 2, 3], [4, 5, 6]])

# 访问数组元素
print(arr1[0])        # 输出：1
print(arr2[1, 2])     # 输出：6

# 数学运算
result = arr1 + 10
print(result)         # 输出：[11 12 13 14 15]

# 切片
sub_array = arr1[1:4]
print(sub_array)      # 输出：[2 3 4]

# 统计运算
mean_value = np.mean(arr1)
print(mean_value)     # 输出：3.0

import numpy as np

# 创建一个全零数组
zeros_array = np.zeros((3, 4))

# 创建一个全一数组
ones_array = np.ones((2, 2))

# 创建一个等差数列
linspace_array = np.linspace(0, 1, 5)  # 生成 [0. 0.25 0.5 0.75 1. ]

# 创建一个随机数组
random_array = np.random.rand(3, 3)  # 生成一个3x3的随机数组

import numpy as np

arr = np.arange(12)  # 创建一个包含0到11的一维数组

# 将一维数组重塑为二维数组
reshaped_arr = arr.reshape(3, 4)

# 改变数组的形状，保持原数组不变
flattened_arr = arr.flatten()

import numpy as np

# 生成服从正态分布的随机数
random_numbers = np.random.normal(loc=0, scale=1, size=(3, 3))

# 随机打乱数组的顺序
arr = np.array([1, 2, 3, 4, 5])
np.random.shuffle(arr)

import numpy as np

arr = np.array([1, 2, 3, 4, 5])

# 使用条件表达式创建新数组
new_array = np.where(arr > 2, arr, 0)  # 大于2的元素保留，小于等于2的变为0

import numpy as np

# 创建一个示例数组
data = np.array([[1, 2, 3], [4, 5, 6]])

# 将数据写入文本文件（以逗号分隔）
np.savetxt('data.txt', data, delimiter=',')

# 将数据写入二进制文件
np.save('data.npy', data)

# 将数据写入压缩的二进制文件
np.savez('data.npz', arr=data, arr2=data*2)

import numpy as np

# 从文本文件读取数据
loaded_data = np.loadtxt('data.txt', delimiter=',')

# 从二进制文件读取数据
loaded_data_binary = np.load('data.npy')

# 从压缩的二进制文件读取数据
loaded_data_compressed = np.load('data.npz')
data_array = loaded_data_compressed['arr']
data_array2 = loaded_data_compressed['arr2']

import numpy as np

arr1 = np.array([1, 2, 3])
arr2 = np.array([4, 5, 6])

# 沿行方向堆叠两个数组
stacked_array = np.vstack((arr1, arr2))

# 沿列方向拼接两个数组
concatenated_array = np.concatenate((arr1, arr2))

Pandas (Python Data Analysis Library) is a popular Python library for data processing and analysis. It provides high-performance, easy-to-use data structures and data analysis tools, making it easier to clean, transform, analyze and visualize data in Python. Here are some of the main features and functionality of Pandas:

1. Data Structures : Pandas introduces two main data structures, DataFrameand Series.

   • DataFrameIt is a two-dimensional tabular data structure, similar to a spreadsheet or SQL table, that can accommodate columns of multiple data types. Each column can have different data types.
   • Seriesis a one-dimensional labeled array, similar to a labeled NumPy array.

2. Data reading : Pandas can easily read various data sources, including CSV, Excel, SQL database, JSON, HTML, and Web API, etc.

3. Data cleaning and processing : Pandas provides rich data operation functions, including missing value processing, data merging, reshaping, filtering, sorting, grouping and aggregation, etc.

4. Data analysis : Pandas supports various data analysis tasks, including statistical description, data visualization, time series analysis, pivot tables, etc.

5. Fast indexing : Pandas DataFrameand Pandas Seriesenable fast data retrieval and indexing by tag or location.

6. Flexible data visualization : Pandas combines visualization libraries such as Matplotlib and Seaborn to make data visualization easy.

7. Data export : Cleaned and analyzed data can be exported to various file formats, including CSV, Excel, SQL database, etc.

8. Time series data : Pandas is very powerful for processing time series data, and can perform date and time parsing, rolling window calculations, etc.

9. High performance : Pandas is designed as a high-performance data processing tool that can handle large data sets.

10. Extensive community support : Due to its popularity and widespread use, Pandas has extensive community support and documentation resources.

Here is a simple example that demonstrates how to create and manipulate data using Pandas:

import pandas as pd

# 创建一个DataFrame
data = {'Name': ['Alice', 'Bob', 'Charlie', 'David', 'Ella'],
        'Age': [25, 30, 35, 28, 22],
        'City': ['New York', 'Los Angeles', 'Chicago', 'Houston', 'Miami']}
df = pd.DataFrame(data)

# 显示DataFrame的前几行数据
print("DataFrame:")
print(df)

# 获取DataFrame的基本信息
print("\nDataFrame Info:")
print(df.info())

# 访问列
print("\nAge Column:")
print(df['Age'])

# 添加新列
df['Salary'] = [50000, 60000, 75000, 48000, 55000]

# 数据过滤
print("\nPeople younger than 30:")
print(df[df['Age'] < 30])

# 数据排序
print("\nSorted by Age:")
print(df.sort_values(by='Age'))

# 数据聚合
print("\nAverage Salary:")
print(df['Salary'].mean())

# 数据分组
grouped = df.groupby('City')
print("\nGrouped by City:")
for city, group in grouped:
    print(city)
    print(group)

# 数据导出为CSV文件
df.to_csv('sample_data1.csv', index=False)

# 从CSV文件导入数据
imported_df = pd.read_csv('sample_data1.csv')
print("\nImported DataFrame:")
print(imported_df)

import pandas as pd

# 从CSV文件读取数据
df = pd.read_csv('data.csv')

# 从Excel文件读取数据
df = pd.read_excel('data.xlsx')

# 从SQL数据库读取数据
import sqlite3
conn = sqlite3.connect('database.db')
query = "SELECT * FROM table_name"
df = pd.read_sql_query(query, conn)

# 选择DataFrame中的一列
column = df['Column_Name']

# 使用条件选择行
filtered_df = df[df['Age'] > 25]

# 使用iloc按位置选择数据
selected_data = df.iloc[0:5, 1:3]

# 处理缺失值
df.dropna()  # 删除包含缺失值的行
df.fillna(value)  # 填充缺失值为指定值

# 数据排序
sorted_df = df.sort_values(by='Column_Name', ascending=False)

# 数据分组和聚合
grouped = df.groupby('Category')
mean_age = grouped['Age'].mean()

# 将DataFrame保存为CSV文件
df.to_csv('output_data.csv', index=False)

# 将DataFrame保存为Excel文件
df.to_excel('output_data.xlsx', index=False)