版权声明：本博客为博主原创博文，可以任意转载。转载请附上博客链接： http://blog.csdn.net/qiqishuang https://blog.csdn.net/qiqishuang/article/details/52153034
Neutron L3 Agent(Layer-3 Networking Extension)作为一种API扩展（通过API来创建router或者floating ip，以提供路由以及NAT的功能），向租户提供了路由和NAT功能。l3扩展包含两种资源：
router：在不同内部子网中转发数据包；通过指定内部网关做NAT。每一个子网对应router上的一个端口，这个端口的ip就是子网的网关。
floating ip：代表一个外部网络的IP，映射到内部网络的端口上。当网络的router:external属性为True时，floating ip才能定义。
这两种资源都对应有不同的属性。支持CRUD操作。
初始化过程分析
既然neutron中支持了l3扩展，那么怎样通过API来创建router或者floating ip，以提供路由以及NAT的功能的呢？
主要有以下几个步骤：
1. 租户通过horizon，nova命令或者自定义的脚本，发送与router或floating ip相关的操作。
2. 这些API请求发送到neutron server，通过neutron提供的API extension相对应。
3. 实现这些API extension的操作，比如说create_router，则由具体的plugin和database来共同完成。
4. plugin会通过rpc机制与计算网络节点上运行的l3 agent来执行l3 转发和NAT的功能。
l3.py
源代码目录：neutron/extensions/l3.py
这个模块中，class RouterPluginBase定义了plugin中需要实现的方法。
class RouterPluginBase(object):
@abc.abstractmethod
def create_router(self, context, router):
pass
@abc.abstractmethod
def update_router(self, context, id, router):
pass
@abc.abstractmethod
def get_router(self, context, id, fields=None):
pass
@abc.abstractmethod
def delete_router(self, context, id):
pass
@abc.abstractmethod
def get_routers(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None, page_reverse=False):
pass
@abc.abstractmethod
def add_router_interface(self, context, router_id, interface_info):
pass
@abc.abstractmethod
def remove_router_interface(self, context, router_id, interface_info):
pass
@abc.abstractmethod
def create_floatingip(self, context, floatingip):
pass
@abc.abstractmethod
def update_floatingip(self, context, id, floatingip):
pass
@abc.abstractmethod
def get_floatingip(self, context, id, fields=None):
pass
@abc.abstractmethod
def delete_floatingip(self, context, id):
pass
@abc.abstractmethod
def get_floatingips(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
pass
def get_routers_count(self, context, filters=None):
raise NotImplementedError()
def get_floatingips_count(self, context, filters=None):
raise NotImplementedError()l3_db.py
源码目录：/neutron/db/l3_db.py
这个模块中，class L3_NAT_dbonly_mixin继承了上面l3模块的class RouterPluginBase，因此在RouterPluginBase中定义的抽象方法就要在这里实现了。
L3_NAT_dbonly_mixin具体方法实现类
class L3_NAT_dbonly_mixin(l3.RouterPluginBase):
"""Mixin class to add L3/NAT router methods to db_base_plugin_v2."""
router_device_owners = (
DEVICE_OWNER_ROUTER_INTF,
DEVICE_OWNER_ROUTER_GW,
DEVICE_OWNER_FLOATINGIP
)
@property
def _core_plugin(self):
return manager.NeutronManager.get_plugin()
def _get_router(self, context, router_id):
try:
router = self._get_by_id(context, Router, router_id)
except exc.NoResultFound:
raise l3.RouterNotFound(router_id=router_id)
return router
def _make_router_dict(self, router, fields=None, process_extensions=True):
res = dict((key, router[key]) for key in CORE_ROUTER_ATTRS)
if router['gw_port_id']:
ext_gw_info = {
'network_id': router.gw_port['network_id'],
'external_fixed_ips': [{'subnet_id': ip["subnet_id"],
'ip_address': ip["ip_address"]}
for ip in router.gw_port['fixed_ips']]}
else:
ext_gw_info = None
res.update({
EXTERNAL_GW_INFO: ext_gw_info,
'gw_port_id': router['gw_port_id'],
})
# NOTE(salv-orlando): The following assumes this mixin is used in a
# class inheriting from CommonDbMixin, which is true for all existing
# plugins.
if process_extensions:
self._apply_dict_extend_functions(l3.ROUTERS, res, router)
return self._fields(res, fields)
def _create_router_db(self, context, router, tenant_id):
"""Create the DB object."""
with context.session.begin(subtransactions=True):
# pre-generate id so it will be available when
# configuring external gw port
router_db = Router(id=(router.get('id') or
uuidutils.generate_uuid()),
tenant_id=tenant_id,
name=router['name'],
admin_state_up=router['admin_state_up'],
status="ACTIVE")
context.session.add(router_db)
return router_db
L3RpcNotifierMixin
class L3RpcNotifierMixin(object):
"""Mixin class to add rpc notifier attribute to db_base_plugin_v2."""
@property
def l3_rpc_notifier(self):
if not hasattr(self, '_l3_rpc_notifier'):
self._l3_rpc_notifier = l3_rpc_agent_api.L3AgentNotifyAPI()
return self._l3_rpc_notifier
@l3_rpc_notifier.setter
def l3_rpc_notifier(self, value):
self._l3_rpc_notifier = value
def notify_router_updated(self, context, router_id,
operation=None):
if router_id:
self.l3_rpc_notifier.routers_updated(
context, [router_id], operation)
def notify_routers_updated(self, context, router_ids,
operation=None, data=None):
if router_ids:
self.l3_rpc_notifier.routers_updated(
context, router_ids, operation, data)
def notify_router_deleted(self, context, router_id):
self.l3_rpc_notifier.router_deleted(context, router_id)
L3_NAT_db_mixin
class L3_NAT_db_mixin(L3_NAT_dbonly_mixin, L3RpcNotifierMixin)
class L3_NAT_db_mixin(L3_NAT_dbonly_mixin, L3RpcNotifierMixin):
"""Mixin class to add rpc notifier methods to db_base_plugin_v2."""
def update_router(self, context, id, router):
router_dict = super(L3_NAT_db_mixin, self).update_router(context,
id, router)
self.notify_router_updated(context, router_dict['id'], None)
return router_dict
def delete_router(self, context, id):
super(L3_NAT_db_mixin, self).delete_router(context, id)
self.notify_router_deleted(context, id)
def notify_router_interface_action(
self, context, router_interface_info, action):
l3_method = '%s_router_interface' % action
super(L3_NAT_db_mixin, self).notify_routers_updated(
context, [router_interface_info['id']], l3_method,
{'subnet_id': router_interface_info['subnet_id']})
mapping = {'add': 'create', 'remove': 'delete'}
notifier = n_rpc.get_notifier('network')
router_event = 'router.interface.%s' % mapping[action]
notifier.info(context, router_event,
{'router_interface': router_interface_info})
def add_router_interface(self, context, router_id, interface_info):
router_interface_info = super(
L3_NAT_db_mixin, self).add_router_interface(
context, router_id, interface_info)
self.notify_router_interface_action(
context, router_interface_info, 'add')
return router_interface_info
def remove_router_interface(self, context, router_id, interface_info):
router_interface_info = super(
L3_NAT_db_mixin, self).remove_router_interface(
context, router_id, interface_info)
self.notify_router_interface_action(
context, router_interface_info, 'remove')
return router_interface_info
def create_floatingip(self, context, floatingip,
initial_status=l3_constants.FLOATINGIP_STATUS_ACTIVE):
floatingip_dict = super(L3_NAT_db_mixin, self).create_floatingip(
context, floatingip, initial_status)
router_id = floatingip_dict['router_id']
self.notify_router_updated(context, router_id, 'create_floatingip')
return floatingip_dict
def update_floatingip(self, context, id, floatingip):
old_floatingip, floatingip = self._update_floatingip(
context, id, floatingip)
router_ids = self._floatingips_to_router_ids(
[old_floatingip, floatingip])
super(L3_NAT_db_mixin, self).notify_routers_updated(
context, router_ids, 'update_floatingip', {})
return floatingip
def delete_floatingip(self, context, id):
floating_ip = self._delete_floatingip(context, id)
self.notify_router_updated(context, floating_ip['router_id'],
'delete_floatingip')
def disassociate_floatingips(self, context, port_id, do_notify=True):
"""Disassociate all floating IPs linked to specific port.
@param port_id: ID of the port to disassociate floating IPs.
@param do_notify: whether we should notify routers right away.
@return: set of router-ids that require notification updates
if do_notify is False, otherwise None.
"""
router_ids = super(L3_NAT_db_mixin, self).disassociate_floatingips(
context, port_id)
if do_notify:
self.notify_routers_updated(context, router_ids)
# since caller assumes that we handled notifications on its
# behalf, return nothing
return
return router_ids
def notify_routers_updated(self, context, router_ids):
super(L3_NAT_db_mixin, self).notify_routers_updated(
context, list(router_ids), 'disassociate_floatingips', {})类注释中写道，Mixin class to add L3/NAT router methods to db_plugin_base_v2。在类L3RpcNotifierMixin的开始，有这样一段代码：
self._l3_rpc_notifier = l3_rpc_agent_api.L3AgentNotifyAPI()
class L3RpcNotifierMixin(object):
"""Mixin class to add rpc notifier attribute to db_base_plugin_v2."""
@property
def l3_rpc_notifier(self):
if not hasattr(self, '_l3_rpc_notifier'):
self._l3_rpc_notifier = l3_rpc_agent_api.L3AgentNotifyAPI()
return self._l3_rpc_notifier说明l3_rpc_notifier，是模块l3_rpc_agent_api中类L3AgentNotifyAPI的一个实例。
l3_rpc_agent_api
/neutron/api/rpc/agentnotifiers/l3_rpc_agent_api.py。
这个类主要用于plugin发送rpc通知给l3 agent。
class L3AgentNotifyAPI(object):
"""API for plugin to notify L3 agent."""
def __init__(self, topic=topics.L3_AGENT):
target = oslo_messaging.Target(topic=topic, version='1.0')
self.client = n_rpc.get_client(target)
def _notification_host(self, context, method, host, use_call=False,
**kwargs):
"""Notify the agent that is hosting the router."""
LOG.debug('Notify agent at %(host)s the message '
'%(method)s', {'host': host,
'method': method})
cctxt = self.client.prepare(server=host)
rpc_method = cctxt.call if use_call else cctxt.cast
rpc_method(context, method, **kwargs)
def _agent_notification(self, context, method, router_ids, operation,
shuffle_agents):
"""Notify changed routers to hosting l3 agents."""
adminContext = context if context.is_admin else context.elevated()
plugin = manager.NeutronManager.get_service_plugins().get(
service_constants.L3_ROUTER_NAT)
state = agentschedulers_db.get_admin_state_up_filter()
for router_id in router_ids:
l3_agents = plugin.get_l3_agents_hosting_routers(
adminContext, [router_id],
admin_state_up=state,
active=True)
if shuffle_agents:
random.shuffle(l3_agents)
for l3_agent in l3_agents:
LOG.debug('Notify agent at %(topic)s.%(host)s the message '
'%(method)s',
{'topic': l3_agent.topic,
'host': l3_agent.host,
'method': method})
cctxt = self.client.prepare(topic=l3_agent.topic,
server=l3_agent.host,
version='1.1')
cctxt.cast(context, method, routers=[router_id])
def _agent_notification_arp(self, context, method, router_id,
operation, data):
"""Notify arp details to l3 agents hosting router."""
if not router_id:
return
adminContext = (context.is_admin and
context or context.elevated())
plugin = manager.NeutronManager.get_service_plugins().get(
service_constants.L3_ROUTER_NAT)
state = agentschedulers_db.get_admin_state_up_filter()
l3_agents = (plugin.
get_l3_agents_hosting_routers(adminContext,
[router_id],
admin_state_up=state,
active=True))
# TODO(murali): replace cast with fanout to avoid performance
# issues at greater scale.
for l3_agent in l3_agents:
log_topic = '%s.%s' % (l3_agent.topic, l3_agent.host)
LOG.debug('Casting message %(method)s with topic %(topic)s',
{'topic': log_topic, 'method': method})
dvr_arptable = {'router_id': router_id,
'arp_table': data}
cctxt = self.client.prepare(topic=l3_agent.topic,
server=l3_agent.host,
version='1.2')
cctxt.cast(context, method, payload=dvr_arptable)
def _notification(self, context, method, router_ids, operation,
shuffle_agents, schedule_routers=True):
"""Notify all the agents that are hosting the routers."""
plugin = manager.NeutronManager.get_service_plugins().get(
service_constants.L3_ROUTER_NAT)
if not plugin:
LOG.error(_LE('No plugin for L3 routing registered. Cannot notify '
'agents with the message %s'), method)
return
if utils.is_extension_supported(
plugin, constants.L3_AGENT_SCHEDULER_EXT_ALIAS):
adminContext = (context.is_admin and
context or context.elevated())
if schedule_routers:
plugin.schedule_routers(adminContext, router_ids)
self._agent_notification(
context, method, router_ids, operation, shuffle_agents)
else:
cctxt = self.client.prepare(fanout=True)
cctxt.cast(context, method, routers=router_ids)
def _notification_fanout(self, context, method, router_id):
"""Fanout the deleted router to all L3 agents."""
LOG.debug('Fanout notify agent at %(topic)s the message '
'%(method)s on router %(router_id)s',
{'topic': topics.L3_AGENT,
'method': method,
'router_id': router_id})
cctxt = self.client.prepare(fanout=True)
cctxt.cast(context, method, router_id=router_id)
def agent_updated(self, context, admin_state_up, host):
self._notification_host(context, 'agent_updated', host,
payload={'admin_state_up': admin_state_up})
def router_deleted(self, context, router_id):
self._notification_fanout(context, 'router_deleted', router_id)
def routers_updated(self, context, router_ids, operation=None, data=None,
shuffle_agents=False, schedule_routers=True):
if router_ids:
self._notification(context, 'routers_updated', router_ids,
operation, shuffle_agents, schedule_routers)
def add_arp_entry(self, context, router_id, arp_table, operation=None):
self._agent_notification_arp(context, 'add_arp_entry', router_id,
operation, arp_table)
def del_arp_entry(self, context, router_id, arp_table, operation=None):
self._agent_notification_arp(context, 'del_arp_entry', router_id,
operation, arp_table)
def router_removed_from_agent(self, context, router_id, host):
self._notification_host(context, 'router_removed_from_agent', host,
payload={'router_id': router_id})
def router_added_to_agent(self, context, router_ids, host):
# need to use call here as we want to be sure agent received
# notification and router will not be "lost". However using call()
# itself is not a guarantee, calling code should handle exceptions and
# retry
self._notification_host(context, 'router_added_to_agent', host,
use_call=True, payload=router_ids)
def routers_updated_on_host(self, context, router_ids, host):
self._notification_host(context, 'routers_updated', host,
routers=router_ids)RPC调用处理
在上面的l3_db.py中，会将涉及router和floating ip的处理读取或者写入到数据中。但是还有一些操作不仅如此，还需要通过rpc（通过调用l3_rpc_agent_api中的函数，这些操作大部分会去 调用routers_updated），通知l3 agent进行处理。
这些需要处理的地方包括（对router，floating_ip，interface的删del、改update操作）：update_router，delete_router，add_router_interface，remove_router_interface，create_floatingip，update_floatingip，delete_floatingip，disassociate_floatingips 等操作。
l3_agent.py
源码目录：neutron/agent/l3_agent.py
l3 agent使用Linux ip协议栈和iptables来实现router和NAT的功能。
这时候，如果在horizon的界面创建一个路由，不进行任何操作的话，plugin只会操作数据库，l3 agent不会作处理。而当update router，如设置外部网关时，l3才会去处理请求。
l3 agent使用service框架启动服务，其manager类为
neutron.agent.l3_agent.L3NATAgentWithStateReport，该类继承自L3NATAgent，主要实现了基于rpc的_report_state向PluginReportStateAPI(topic为q-plugin)汇报状态信息，这些信息由各个plugin来处理（比如ml2中通过start_rpc_listeners来注册该topic的消费者）。
L3NATAgent类是最主要的L3 Manager类，该类继承关系为
class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager)；
FWaaSL3AgentRpcCallback主要是加载防火墙驱动FWaaS Driver，并创建RPC与Plugin通信。
再来看L3NATAgent的创建过程：
class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
ha.AgentMixin,
dvr.AgentMixin,
manager.Manager):
"""Manager for L3NatAgent
API version history:
1.0 initial Version
1.1 changed the type of the routers parameter
to the routers_updated method.
It was previously a list of routers in dict format.
It is now a list of router IDs only.
Per rpc versioning rules, it is backwards compatible.
1.2 - DVR support: new L3 agent methods added.
- add_arp_entry
- del_arp_entry
Needed by the L3 service when dealing with DVR
"""
L3NATAgent
该类是最主要的L3 Manager类
其中：
self.plugin_rpc = L3PluginApi(topics.L3PLUGIN, host)该self.plugin_rpc会处理neutron-server转发过来的请求，这个请求是通过service_plugins的方式处理的：
neutron.service_plugins =
dummy = neutron.tests.unit.dummy_plugin:DummyServicePlugin
router = neutron.services.l3_router.l3_router_plugin:L3RouterPlugin
bigswitch_l3 = neutron.plugins.bigswitch.l3_router_plugin:L3RestProxy
brocade_vyatta_l3 = neutron.services.l3_router.brocade.vyatta.vrouter_neutron_plugin:VyattaVRouterPlugin
brocade_mlx_l3 = neutron.services.l3_router.brocade.mlx.l3_router_plugin:BrocadeRouterPlugin
firewall = neutron_fwaas.services.firewall.fwaas_plugin:FirewallPlugin
fsl_firewall = neutron_fwaas.services.firewall.freescale.fwaas_plugin:FirewallPlugin
lbaas = neutron_lbaas.services.loadbalancer.plugin:LoadBalancerPlugin
vpnaas = neutron_vpnaas.services.vpn.plugin:VPNDriverPlugin
metering = neutron.services.metering.metering_plugin:MeteringPlugin
neutron.services.firewall.fwaas_plugin.FirewallPlugin = neutron_fwaas.services.firewall.fwaas_plugin:FirewallPlugin
neutron.services.loadbalancer.plugin.LoadBalancerPlugin = neutron_lbaas.services.loadbalancer.plugin:LoadBalancerPlugin
neutron.services.vpn.plugin.VPNDriverPlugin = neutron_vpnaas.services.vpn.plugin:VPNDriverPlugin
ibm_l3 = neutron.services.l3_router.l3_sdnve:SdnveL3ServicePlugin
qos = neutron.services.qos.qos_plugin:QoSPlugin
其中：
文件路径： neutron/agent/l3/agent.py
L3NATAgentWithStateReport
class L3NATAgentWithStateReport(L3NATAgent):
def __init__(self, host, conf=None):
super(L3NATAgentWithStateReport, self).__init__(host=host, conf=conf)
self.state_rpc = agent_rpc.PluginReportStateAPI(topics.PLUGIN)
self.agent_state = {
'binary': 'neutron-l3-agent',
'host': host,
'topic': topics.L3_AGENT,
'configurations': {
'agent_mode': self.conf.agent_mode,
'use_namespaces': self.conf.use_namespaces,
'router_id': self.conf.router_id,
'handle_internal_only_routers':
self.conf.handle_internal_only_routers,
'external_network_bridge': self.conf.external_network_bridge,
'gateway_external_network_id':
self.conf.gateway_external_network_id,
'interface_driver': self.conf.interface_driver,
'log_agent_heartbeats': self.conf.AGENT.log_agent_heartbeats},
'start_flag': True,
'agent_type': l3_constants.AGENT_TYPE_L3}
report_interval = self.conf.AGENT.report_interval
if report_interval:
# self.heartbeat会循环检测从plugin发送过来的rpc请求：
self.heartbeat = loopingcall.FixedIntervalLoopingCall(
self._report_state)
self.heartbeat.start(interval=report_interval)
_process_routers_loop
def _process_routers_loop(self):
LOG.debug("Starting _process_routers_loop")
pool = eventlet.GreenPool(size=8)
while True:
pool.spawn_n(self._process_router_update)_process_routers_loop
如果有rpc请求过来，即需要更新路由信息，或者添加路由子接口，创建floating ip等操作，都会在这里执行。这个函数里会去调用_process_routers_loop函数，在_process_routers_loop函数中会去创建绿色线程，执行_process_router_update函数。可以说，l3 agent调用网络设备的工作都会在_process_router_update中进行。
_process_router_update
def _process_router_update(self):
for rp, update in self._queue.each_update_to_next_router():
LOG.debug("Starting router update for %s, action %s, priority %s",
update.id, update.action, update.priority)
if update.action == queue.PD_UPDATE:
self.pd.process_prefix_update()
LOG.debug("Finished a router update for %s", update.id)
continue
router = update.router
if update.action != queue.DELETE_ROUTER and not router:
try:
update.timestamp = timeutils.utcnow()
routers = self.plugin_rpc.get_routers(self.context,
[update.id])
except Exception:
msg = _LE("Failed to fetch router information for '%s'")
LOG.exception(msg, update.id)
self._resync_router(update)
continue
if routers:
router = routers[0]
if not router:
removed = self._safe_router_removed(update.id)
if not removed:
self._resync_router(update)
else:
# need to update timestamp of removed router in case
# there are older events for the same router in the
# processing queue (like events from fullsync) in order to
# prevent deleted router re-creation
rp.fetched_and_processed(update.timestamp)
LOG.debug("Finished a router update for %s", update.id)
continue
try:
self._process_router_if_compatible(router)
except n_exc.RouterNotCompatibleWithAgent as e:
LOG.exception(e.msg)
# Was the router previously handled by this agent?
if router['id'] in self.router_info:
LOG.error(_LE("Removing incompatible router '%s'"),
router['id'])
self._safe_router_removed(router['id'])
except Exception:
msg = _LE("Failed to process compatible router '%s'")
LOG.exception(msg, update.id)
self._resync_router(update)
continue
LOG.debug("Finished a router update for %s", update.id)
rp.fetched_and_processed(update.timestamp)
_process_router_update函数所做的工作有：
1.处理内部接口
这个是在router添加和删除子接口时工作。它会调用internal_network_added和internal_network_removed这个两个函数。
在internal_network_added和internal_network_removed这个两个函数会去调用OVSInterfaceDriver的plug和unplug 函数，这两个函数最终会用ip link 和ip addr的命令去处理接口和ip地址。
2.处理外部网关
router添加和删除外部网关。调用external_gateway_added和external_gateway_removed函数，同样也会调用plug和unplug函数，用ip link 和ip addr的命令进行最终处理
3.为外部网关做SNAT
调用_handle_router_snat_rules函数，使用iptables来加链和删除链。
在我的测试网络中，router上有3个接口，外部网关地址为192.168.39.2，内部两个子网的网关为10.1.0.1，10.2.0.1。iptables规则如下：
iptables -t nat -A POSTROUTING ! -i qg-fcb1a762-1f ! -o qg-fcb1a762-1f -m conntrack ! --ctstate DNAT -j ACCEPT
iptables -t nat -A snat -s 10.2.0.1/24 -j SNAT --to-source 192.168.39.2
iptables -t nat -A snat -s 10.1.0.1/24 -j SNAT --to-source 192.168.39.2qg-fcb1a762-1f为外部网关接口的索引，使用ip netns exec $namespace ip link list可查看。
4.为floating ip做SNAT/DNAT
和浮动IP相关，如创建，更新，删除，绑定到一个云主机的接口，解绑定等。
不同neutron版本这部分的处理不同，这里是基于Icehouse rc1版本的，在havava stable版本，只有一个函数来处理iptables规则和floating ip。
process_router_floating_ip_nat_rules ：当floating ip与云主机绑定时，会先清除已有的floating_ip规则，再加上要添加的iptables规则,同时重新加载清除的iptables规则。
比如，一个云主机10.1.0.2上绑定了一个floating ip（192.168.39.5）。那么最终会在iptable不同的链中添加iptables规则，float-snat为neutron自定义链。
iptables -t nat -A PREROUTING -d 192.168.39.5 -j DNAT --to 10.1.0.2
iptables -t nat -A OUTPUT -d 192.168.39.5 -j DNAT --to 10.1.0.2
iptables -t nat -A float-snat -s 10.1.0.2 -j SNAT --to 192.168.39.5process_router_floating_ip_addresses：
将floating ip和云主机绑定时，使用ip addr add命令添加ip地址。
解除floating ip和云主机绑定时，使用ip addr del命令将floating ip删除。
参考：
about云： http://www.aboutyun.com/thread-9529-1-1.html