OpenStack-neutron组件部署步骤

其他 2021-03-20 05:42:00 阅读次数: 0

文章目录

创建数据库neutron，并进行授权

[root@ct ~]# mysql -u root -p
MariaDB [(none)]> CREATE DATABASE neutron;
MariaDB [(none)]> GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'localhost' IDENTIFIED BY 'NEUTRON_DBPASS';
MariaDB [(none)]> GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'%' IDENTIFIED BY 'NEUTRON_DBPASS';
MariaDB [(none)]> flush privileges;
MariaDB [(none)]> exit

● 创建neutron用户，用于在keystone做认证

 [root@ct ~]# openstack user create --domain default --password NEUTRON_PASS neutron

● 将neutron用户添加到service项目中拥有管理员权限

[root@ct ~]# openstack role add --project service --user neutron admin

● 创建network服务，服务类型为network

[root@ct ~]# openstack service create --name neutron --description "OpenStack Networking" network

● 注册API到neutron服务，给neutron服务关联端口，即添加endpoint

[root@ct ~]# openstack endpoint create --region RegionOne network public http://ct:9696
[root@ct ~]# openstack endpoint create --region RegionOne network internal http://ct:9696	
[root@ct ~]# openstack endpoint create --region RegionOne network admin http://ct:9696

● 安装提供者网络(桥接)

● ebtables包是用来管理iptables规则的

[root@ct ~]# yum -y install openstack-neutron openstack-neutron-ml2 openstack-neutron-linuxbridge ebtables conntrack-tools

● 更改主配置文件

#修改主配置文件neutron.conf
cp -a /etc/neutron/neutron.conf{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/neutron.conf.bak > /etc/neutron/neutron.conf
openstack-config --set /etc/neutron/neutron.conf database connection mysql+pymysql://neutron:NEUTRON_DBPASS@ct/neutron
openstack-config --set /etc/neutron/neutron.conf DEFAULT core_plugin ml2
openstack-config --set /etc/neutron/neutron.conf DEFAULT service_plugins router
openstack-config --set  /etc/neutron/neutron.conf DEFAULT allow_overlapping_ips true
openstack-config --set /etc/neutron/neutron.conf DEFAULT transport_url rabbit://openstack:RABBIT_PASS@ct
openstack-config --set /etc/neutron/neutron.conf DEFAULT auth_strategy keystone
openstack-config --set /etc/neutron/neutron.conf DEFAULT notify_nova_on_port_status_changes true
openstack-config --set /etc/neutron/neutron.conf DEFAULT notify_nova_on_port_data_changes true
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken www_authenticate_uri http://ct:5000
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken auth_url http://ct:5000
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken memcached_servers ct:11211
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken auth_type password
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken project_domain_name default
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken user_domain_name default
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken project_name service
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken username neutron
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken password NEUTRON_PASS
openstack-config --set /etc/neutron/neutron.conf oslo_concurrency lock_path /var/lib/neutron/tmp
openstack-config --set  /etc/neutron/neutron.conf nova  auth_url http://ct:5000
openstack-config --set  /etc/neutron/neutron.conf nova  auth_type password
openstack-config --set  /etc/neutron/neutron.conf nova  project_domain_name default
openstack-config --set  /etc/neutron/neutron.conf nova  user_domain_name default
openstack-config --set  /etc/neutron/neutron.conf nova  region_name RegionOne
openstack-config --set  /etc/neutron/neutron.conf nova  project_name service
openstack-config --set  /etc/neutron/neutron.conf nova  username nova
openstack-config --set  /etc/neutron/neutron.conf nova  password NOVA_PASS

#查看配置文件
[root@ct ~]# cd /etc/neutron/
[root@ct neutron]# cat neutron.conf
[DEFAULT]
core_plugin = ml2						#启用二层网络插件
service_plugins = router				#启用三层网络插件
allow_overlapping_ips = true
transport_url = rabbit://openstack:RABBIT_PASS@ct	 #配置rabbitmq连接
auth_strategy = keystone			                 #认证的方式:keystone
notify_nova_on_port_status_changes = true			 #当网络接口发生变化时，通知给计算节点	
notify_nova_on_port_data_changes = true			     #当端口数据发生变化，通知计算节点
[cors]
[database]						        #配置数据库连接
connection = mysql+pymysql://neutron:NEUTRON_DBPASS@ct/neutron
[keystone_authtoken]					#配置keystone认证信息
www_authenticate_uri = http://ct:5000
auth_url = http://ct:5000
memcached_servers = ct:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = neutron
password = NEUTRON_PASS
[oslo_concurrency]						#配置锁路径
lock_path = /var/lib/neutron/tmp
[oslo_messaging_amqp]
[oslo_messaging_kafka]
[oslo_messaging_notifications]
[oslo_messaging_rabbit]
[oslo_middleware]
[oslo_policy]
[privsep]
[ssl]
[nova]							         #neutron需要给nova返回数据
auth_url = http://ct:5000				 #到keystone认证nova
auth_type = password
project_domain_name = default
user_domain_name = default
region_name = RegionOne
project_name = service
username = nova						    #通过nova的用户名和密码到keystone验证nova的token
password = NOVA_PASS

● 修改 ML2 plugin 配置文件 ml2_conf.ini

#修改参数
cp -a /etc/neutron/plugins/ml2/ml2_conf.ini{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/plugins/ml2/ml2_conf.ini.bak > /etc/neutron/plugins/ml2/ml2_conf.ini
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 type_drivers  flat,vlan,vxlan
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 tenant_network_types vxlan
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 mechanism_drivers  linuxbridge,l2population
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 extension_drivers  port_security
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_flat flat_networks  provider
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_vxlan vni_ranges 1:1000
openstack-config --set /etc/neutron/plugins/ml2/ml2_conf.ini securitygroup enable_ipset  true


#查看配置文件
[root@ct ~]# cd /etc/neutron/plugins/ml2
[root@ct ml2]# cat ml2_conf.ini
[DEFAULT]

[ml2]
type_drivers = flat,vlan,vxlan	 #配置类型驱动；单一扁平网络(桥接)和vlan；让二层网络支持桥接，支持基于vlan做子网划分
tenant_network_types = vxlan	 #租户网络类型(vxlan）
mechanism_drivers = linuxbridge,l2population		#启用Linuxbridge和l2机制，(l2population机制是为了简化网络通信拓扑，减少网络广播)：
extension_drivers = port_security		#启用端口安全扩展驱动程序，基于iptables实现访问控制；但配置了扩展安全组会导致一些端口限制，造成一些服务无法启动 

[ml2_type_flat]
flat_networks = provider				#配置公共虚拟网络为flat网络

[ml2_type_vxlan]
vni_ranges = 1:1000				        #为私有网络配置VXLAN网络识别的网络范围

[securitygroup]
enable_ipset = true					    #启用 ipset 增加安全组的方便性

● 修改 linux bridge network provider 配置文件

#Linux网桥
[root@ct ~]# cp -a /etc/neutron/plugins/ml2/linuxbridge_agent.ini{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/plugins/ml2/linuxbridge_agent.ini.bak > /etc/neutron/plugins/ml2/linuxbridge_agent.ini
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini linux_bridge physical_interface_mappings  provider:eth1		###eth1网卡名称
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini vxlan enable_vxlan  true
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini vxlan local_ip 192.168.100.11	##控制节点IP地址	
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini vxlan l2_population true
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini securitygroup enable_security_group  true
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini securitygroup firewall_driver  neutron.agent.linux.iptables_firewall.IptablesFirewallDriver

#查看配置文件
[root@ct ~]# cd /etc/neutron/plugins/ml2/
[root@ct ml2]# cat linuxbridge_agent.ini
[DEFAULT]

[linux_bridge]
physical_interface_mappings = provider:eth1		#指定上个文件中的桥接网络名称，与eth0物理网卡做关联，后期给虚拟机分配external网络，就可以通过eth0上外网；物理网卡有可能是bind0、br0等

[vxlan]				#启用VXLAN覆盖网络，配置覆盖网络的物理网络接口的IP地址，启用layer－2 population
enable_vxlan = true			#允许用户创建自定义网络(3层网络)
local_ip = 192.168.100.11
l2_population = true

[securitygroup]		#启用安全组并配置 Linux 桥接 iptables 防火墙驱动
enable_security_group = true
firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver

● 修改内核参数

[root@ct ~]# echo 'net.bridge.bridge-nf-call-iptables=1' >> /etc/sysctl.conf
[root@ct ~]# echo 'net.bridge.bridge-nf-call-ip6tables=1' >> /etc/sysctl.conf
[root@ct ~]# modprobe br_netfilter	 #表示向内核加入参数
[root@ct ~]# sysctl -p

在这里插入图片描述

● 配置Linuxbridge接口驱动和外部网络网桥

cp -a /etc/neutron/l3_agent.ini{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/l3_agent.ini.bak > /etc/neutron/l3_agent.ini
openstack-config --set /etc/neutron/l3_agent.ini DEFAULT interface_driver linuxbridge

#查看文件
cd /etc/neutron
cat l3_agent.ini
[DEFAULT]
interface_driver = linuxbridge

● 修改dhcp_agent 配置文件

[root@ct ~]# cp -a /etc/neutron/dhcp_agent.ini{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/dhcp_agent.ini.bak > /etc/neutron/dhcp_agent.ini 
openstack-config --set /etc/neutron/dhcp_agent.ini DEFAULT interface_driver linuxbridge
openstack-config --set /etc/neutron/dhcp_agent.ini DEFAULT dhcp_driver neutron.agent.linux.dhcp.Dnsmasq
openstack-config --set /etc/neutron/dhcp_agent.ini DEFAULT enable_isolated_metadata true

● 修改dhcp_agent的配置文件内容

[root@ct ~]# cd /etc/neutron/
[root@ct neutron]# cat dhcp_agent.ini
[DEFAULT]
interface_driver = linuxbridge	                #指定默认接口驱动为linux网桥
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq	#指定DHCP驱动
enable_isolated_metadata = true			        #开启iso元数据

● 配置元数据代理、用于配置桥接与自服务网络的通用配置

[root@ct ~]# cp -a /etc/neutron/metadata_agent.ini{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/metadata_agent.ini.bak > /etc/neutron/metadata_agent.ini
openstack-config --set /etc/neutron/metadata_agent.ini DEFAULT nova_metadata_host ct
openstack-config --set /etc/neutron/metadata_agent.ini DEFAULT metadata_proxy_shared_secret METADATA_SECRET

● 配置文件内容

[root@ct ~]# cd /etc/neutron/
[root@ct neutron]# cat metadata_agent.ini
[DEFAULT]
nova_metadata_host = ct
metadata_proxy_shared_secret = METADATA_SECRET
[cache]

● 修改nova配置文件，用于neutron交互

#修改CT配置文件
openstack-config --set /etc/nova/nova.conf neutron url http://ct:9696
openstack-config --set /etc/nova/nova.conf neutron auth_url http://ct:5000
openstack-config --set /etc/nova/nova.conf neutron auth_type password
openstack-config --set /etc/nova/nova.conf neutron project_domain_name default
openstack-config --set /etc/nova/nova.conf neutron user_domain_name default
openstack-config --set /etc/nova/nova.conf neutron region_name RegionOne
openstack-config --set /etc/nova/nova.conf neutron project_name service
openstack-config --set /etc/nova/nova.conf neutron username neutron
openstack-config --set /etc/nova/nova.conf neutron password NEUTRON_PASS
openstack-config --set /etc/nova/nova.conf neutron service_metadata_proxy true
openstack-config --set /etc/nova/nova.conf neutron metadata_proxy_shared_secret METADATA_SECRET

● 查看配置文件

[root@ct ~]# cd /etc/nova/
[root@ct nova]# cat nova.conf
[DEFAULT]
enabled_apis = osapi_compute,metadata
my_ip = 192.168.100.11
use_neutron = true
firewall_driver = nova.virt.firewall.NoopFirewallDriver
transport_url = rabbit://openstack:RABBIT_PASS@ct
[api]
auth_strategy = keystone
[api_database]
connection = mysql+pymysql://nova:NOVA_DBPASS@ct/nova_api
[barbican]
[cache]
[cinder]
[compute]
[conductor]
[console]
[consoleauth]
[cors]
[database]
connection = mysql+pymysql://nova:NOVA_DBPASS@ct/nova
[devices]
[ephemeral_storage_encryption]
[filter_scheduler]
[glance]
api_servers = http://ct:9292
[guestfs]
[healthcheck]
[hyperv]
[ironic]
[key_manager]
[keystone]
[keystone_authtoken]
auth_url = http://ct:5000/v3
memcached_servers = ct:11211
auth_type = password
project_domain_name = Default
user_domain_name = Default
project_name = service
username = nova
password = NOVA_PASS
[libvirt]
[metrics]
[mks]
[neutron]
url = http://ct:9696
auth_url = http://ct:5000
auth_type = password
project_domain_name = default
user_domain_name = default
region_name = RegionOne
project_name = service
username = neutron
password = NEUTRON_PASS
service_metadata_proxy = true
metadata_proxy_shared_secret = METADATA_SECRET
[notifications]
[osapi_v21]
[oslo_concurrency]
lock_path = /var/lib/nova/tmp
[oslo_messaging_amqp]
[oslo_messaging_kafka]
[oslo_messaging_notifications]
[oslo_messaging_rabbit]
[oslo_middleware]
[oslo_policy]
[pci]
[placement]
region_name = RegionOne
project_domain_name = Default
project_name = service
auth_type = password
user_domain_name = Default
auth_url = http://ct:5000/v3
username = placement
password = PLACEMENT_PASS
[powervm]
[privsep]
[profiler]
[quota]
[rdp]
[remote_debug]
[scheduler]
discover_hosts_in_cells_interval = 300
[serial_console]
[service_user]
[spice]
[upgrade_levels]
[vault]
[vendordata_dynamic_auth]
[vmware]
[vnc]
enabled = true
server_listen =  $my_ip
server_proxyclient_address =  $my_ip
[workarounds]
[wsgi]
[xenserver]
[xvp]
[zvm]


[placement_database]
connection = mysql+pymysql://placement:PLACEMENT_DBPASS@ct/placement

● 创建ML2插件文件符号连接

● 网络服务初始化脚本需要/etc/neutron/plugin.ini指向ML2插件配置文件的符号链接

[root@ct ~]# ln -s /etc/neutron/plugins/ml2/ml2_conf.ini /etc/neutron/plugin.ini

● 初始化数据库

[root@ct ~]# su -s /bin/sh -c "neutron-db-manage --config-file /etc/neutron/neutron.conf \
--config-file /etc/neutron/plugins/ml2/ml2_conf.ini upgrade head" neutron

● 重启计算节点nova-api服务

[root@ct ~]# systemctl restart openstack-nova-api.service

● 开启neutron服务、设置开机自启动

[root@ct ~]# systemctl enable neutron-server.service \
neutron-linuxbridge-agent.service neutron-dhcp-agent.service \
neutron-metadata-agent.service
[root@ct ~]# systemctl start neutron-server.service \
neutron-linuxbridge-agent.service neutron-dhcp-agent.service \
neutron-metadata-agent.service
[root@ct ~]# netstat -anutp |grep 9696
tcp        0      0 0.0.0.0:9696            0.0.0.0:*               LISTEN      63486/server.log

● 因为配置了第三层L3网络服务、所以需要启动第三层服务

[root@ct ~]# systemctl enable neutron-l3-agent.service
[root@ct ~]# systemctl restart neutron-l3-agent.service

【c1 节点操作】（c2节点与c1节点操作相同）

[root@c1 ~]# yum -y install openstack-neutron-linuxbridge ebtables ipset conntrack-tools
#ipset：iptables的扩展，允许匹配规则的集合而不仅仅是一个IP

● 修改neutron.conf文件

【修改配置文件（C1、C2)】
cp -a /etc/neutron/neutron.conf{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/neutron.conf.bak > /etc/neutron/neutron.conf
openstack-config --set /etc/neutron/neutron.conf DEFAULT transport_url rabbit://openstack:RABBIT_PASS@ct
openstack-config --set /etc/neutron/neutron.conf DEFAULT auth_strategy keystone
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken www_authenticate_uri http://ct:5000
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken auth_url http://ct:5000
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken memcached_servers ct:11211
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken auth_type password
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken project_domain_name default
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken user_domain_name default
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken project_name service
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken username neutron
openstack-config --set /etc/neutron/neutron.conf keystone_authtoken password NEUTRON_PASS
openstack-config --set /etc/neutron/neutron.conf oslo_concurrency lock_path /var/lib/neutron/tmp


【查看配置文件】
[root@c1 ~]# cd /etc/neutron/
[root@c1 neutron]# cat neutron.conf

[DEFAULT]				#neutron的server端与agent端通讯也是通过rabbitmq进行通讯的
transport_url = rabbit://openstack:RABBIT_PASS@ct
auth_strategy = keystone		#认证策略：keystone
[cors]
[database]

[keystone_authtoken]			#指定keystone认证的信息
www_authenticate_uri = http://ct:5000
auth_url = http://ct:5000
memcached_servers = ct:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = neutron
password = NEUTRON_PASS

[oslo_concurrency]			#配置锁路径（管理线程库）
lock_path = /var/lib/neutron/tmp

[oslo_messaging_amqp]
[oslo_messaging_kafka]
[oslo_messaging_notifications]
[oslo_messaging_rabbit]
[oslo_middleware]
[oslo_policy]
[privsep]
[ssl]

● 配置Linux网桥代理

【修改C1、C2 Linuxbridge网桥配置文件】
cp -a /etc/neutron/plugins/ml2/linuxbridge_agent.ini{
    
    ,.bak}
grep -Ev '^$|#' /etc/neutron/plugins/ml2/linuxbridge_agent.ini.bak > /etc/neutron/plugins/ml2/linuxbridge_agent.ini
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini linux_bridge physical_interface_mappings  provider:eth1
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini vxlan enable_vxlan  true
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini vxlan local_ip 192.168.100.12      #此处需要修改为对应节点得内部IP即VM1的IP
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini vxlan l2_population true
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini securitygroup enable_security_group  true
openstack-config --set /etc/neutron/plugins/ml2/linuxbridge_agent.ini securitygroup firewall_driver  neutron.agent.linux.iptables_firewall.IptablesFirewallDriver


[root@c1 ~]# cd /etc/neutron/plugins/ml2/
[root@c1 ml2]# cat linuxbridge_agent.ini
[DEFAULT]
[linux_bridge]
physical_interface_mappings = provider:eth1
● 直接将node节点external网络绑定在当前节点的指定的物理网卡，不需要node节点配置网络名称，node节点只需要接收controller节点指令即可；controller节点上配置的external网络名称是针对整个openstack环境生效的，所以指定external网络绑定在当前node节点的eth0物理网卡上(也可能是bind0或br0)

[vxlan]
enable_vxlan = true					#开启Vxlan网络
local_ip = 192.168.100.12
l2_population = true				#L2 Population 是用来提高 VXLAN 网络扩展能力的组件

[securitygroup]
enable_security_group = true		#开启安全组
firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver	#指定安全组驱动文件

● 修改内核

echo 'net.bridge.bridge-nf-call-iptables=1' >> /etc/sysctl.conf		#允许虚拟机的数据通过物理机出去
echo 'net.bridge.bridge-nf-call-ip6tables=1' >> /etc/sysctl.conf
modprobe br_netfilter		#modprobe：用于向内核中加载模块或者从内核中移除模块。modprobe -r 表示移除
sysctl -p

在这里插入图片描述

● 修改nova.conf配置文件

【修改nova.conf配置文件的neutron字段-C1、C2节点】
openstack-config --set /etc/nova/nova.conf neutron auth_url http://ct:5000
openstack-config --set /etc/nova/nova.conf neutron auth_type password
openstack-config --set /etc/nova/nova.conf neutron project_domain_name default
openstack-config --set /etc/nova/nova.conf neutron user_domain_name default
openstack-config --set /etc/nova/nova.conf neutron region_name RegionOne
openstack-config --set /etc/nova/nova.conf neutron project_name service
openstack-config --set /etc/nova/nova.conf neutron username neutron
openstack-config --set /etc/nova/nova.conf neutron password NEUTRON_PASS

#以下为修改字段内容
[neutron]
auth_url = http://ct:5000
auth_type = password
project_domain_name = default
user_domain_name = default
region_name = RegionOne
project_name = service
username = neutron
password = NEUTRON_PASS

● C2节点部署neutron服务（与C1节点相同）

● 验证服务组件【ct节点】

[root@ct ~]# openstack extension list --network
[root@ct ~]# openstack network agent list

在这里插入图片描述

● 小结

三个节点都需要配置neutron网络，以下为部署思路：
配置neutron组件的用户、认证、endpoint
设置提供者provider网络（这里是桥接模式）
① 配置二层网络
② 配置网桥（插件）
③ 优化内核
④ 配置网桥接口与外部对接
⑤ 修改DHCP配置（修改配置文件、代理）
⑥ 配置网桥与内部组件的配置（修改配置文件、代理）
设置neutron与nova对接的配置
其中C1、C2节点配置相同

猜你喜欢

转载自blog.csdn.net/Gengchenchen/article/details/114999874
今日推荐
TIOBE 5 月榜单：Fortran “复活”进入 Top 10
GCC 14.1 发布
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
基本数据类型封装类比较 Java源码解读(一) 8种基本类型对应的封装类型
JS实现无缝滚动上
深入解析HashMap原理（基于JDK1.8）
mysql的连接池
关于.htc
linux下的ubuntu12.04图形界面
【数论】好推不好记的扩展欧几里德
设备树详解
cscope + tags 简单设置
xml学习
每日归档
更多
2024-05-09(35)
2024-05-08(42)
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022