OVS源码connmgr_run分析

 

 

connmgr即connect manager连接管理器，主要完成OVS网桥的连接管理。每一个网桥ofproto都有一个connmgr实体来管理连接。connmgr主要完成两种连接管理，一种是主动连接（即与控制器连接，作为客户端），一种是被动连接（主要提供ofctl等工具连接请求服务和snoop服务，作为服务端）。connmgr结构体中，all_conns是主动连接，services和snoops等提供被动连接服务。与connmgr_run流程相关的结构体如下图：

一个connmgr对应多个ofconn（如一个网桥连接多个控制器，则需要建立相应多个ofconn实例；和ofctl等工具连接时也要建立ofconn实例）；一个ofconn对应一个rconn；一个rconn对应多个vconn(因为和控制器建立连接外，可能还会实现snoop功能，即监测并复制交互的of消息，则一个rconn会对应一个vconn和一个vconn结构体类型的数组monitor。

connmgr_run主要包含in_band_run, ofconn_run，以及一些与被动连接相关的流程。

connmgr_run(struct connmgr *mgr,
void (*handle_openflow)(struct ofconn *,
const struct ofpbuf *ofp_msg))
OVS_EXCLUDED(ofproto_mutex)
{
struct ofconn *ofconn, *next_ofconn;
struct ofservice *ofservice;
size_t i;

if (mgr->in_band) {
if (!in_band_run(mgr->in_band)) {
in_band_destroy(mgr->in_band);
mgr->in_band = NULL;
}
}

LIST_FOR_EACH_SAFE (ofconn, next_ofconn, node, &mgr->all_conns) {
ofconn_run(ofconn, handle_openflow);
}

/*When too many flow monitor notifications back up in the transmit buffer,
* we pause the transmission of further notifications. These members track
* the flow control state. */
ofmonitor_run(mgr);

/* Fail-open maintenance. Do this after processing the ofconns since
* fail-open checks the status of the controller rconn. */
if (mgr->fail_open) {
fail_open_run(mgr->fail_open);
}

HMAP_FOR_EACH (ofservice, node, &mgr->services) {
struct vconn *vconn;
int retval;

/*最终将调用pstream_class->accept实现与punix，pssl，ptcp等被动连接*/
retval = pvconn_accept(ofservice->pvconn, &vconn);
if (!retval) {
struct rconn *rconn;
char *name;

/* 对被动连接创建默认rconn实例 */
rconn = rconn_create(ofservice->probe_interval, 0, ofservice->dscp,
vconn_get_allowed_versions(vconn));
name = ofconn_make_name(mgr, vconn_get_name(vconn));
/*断开rconn现有连接，并将rconn与被动连接vconn相连*/
rconn_connect_unreliably(rconn, vconn, name);
free(name);

ovs_mutex_lock(&ofproto_mutex);

/*根据rconn对被动连接创建ofconn实例*/
ofconn = ofconn_create(mgr, rconn, OFCONN_SERVICE,
ofservice->enable_async_msgs);
ovs_mutex_unlock(&ofproto_mutex);

ofconn_set_rate_limit(ofconn, ofservice->rate_limit,
ofservice->burst_limit);
} else if (retval != EAGAIN) {
VLOG_WARN_RL(&rl, "accept failed (%s)", ovs_strerror(retval));
}
}

for (i = 0; i < mgr->n_snoops; i++) {
struct vconn *vconn;
int retval;

/*为snoops创建vconn*/
retval = pvconn_accept(mgr->snoops[i], &vconn);
if (!retval) {
/*snooper与ofservice不同在于snooper不用创建ofconn与rconn实例
*而是在所有的ofconn中根据ofconn->role等级挑选最佳的ofconn
*然后将snoops的vconn加入到最佳ofconn的monitor中，对该rconn进行监听*/
add_snooper(mgr, vconn);
} else if (retval != EAGAIN) {
VLOG_WARN_RL(&rl, "accept failed (%s)", ovs_strerror(retval));
}
}
}

in_band_run：因为OpenvSwitch不仅仅是一个OpenFlow Switch，它的流表组成除了of流表外，还有其他一些（隐藏）流表。这些隐藏流表是由于默认交换机和控制器在同一网络中（in-band），因此要保证两者互通。要关闭默认的inband可以通过“ovs-vsctl set controller br0 connection-mode=out-of-band”

in_band_run(struct in_band *ib)
{
uint64_t ofpacts_stub[ 128 / 8];
struct ofpbuf ofpacts;

struct in_band_rule *rule, *next;

...

refresh_local(ib); //刷新本地MAC
refresh_remotes(ib); //刷新远端MAC

update_rules(ib); //处理本地local port以及所有远端port的ARP，DHCP，TCP等协议

HMAP_FOR_EACH_SAFE (rule, next, hmap_node, &ib->rules) {
switch (rule->op) {
case ADD:
ofproto_add_flow(ib->ofproto, &rule->match, rule->priority,
ofpacts.data, ofpacts.size);
break;

case DEL:
ovs_mutex_lock(&ofproto_mutex);
ofproto_delete_flow(ib->ofproto, &rule->match, rule->priority);
ovs_mutex_unlock(&ofproto_mutex);
hmap_remove(&ib->rules, &rule->hmap_node);
free(rule);
break;
}
}

ofpbuf_uninit(&ofpacts);

return ib->n_remotes || !hmap_is_empty(&ib->rules);
}

ofconn_run：对与connmgr所有的主动连接all_conns产生的ofconn进行处理，主要完成网桥与控制器连接的状态转移管理和of消息的接收处理等。

ofconn_run(struct ofconn *ofconn,
void (*handle_openflow)(struct ofconn *,
const struct ofpbuf *ofp_msg))
{
struct connmgr *mgr = ofconn->connmgr;
size_t i;

for (i = 0; i < N_SCHEDULERS; i++) {
       struct ovs_list txq;


       pinsched_run(ofconn->schedulers[i], &txq); //处理OFPT_PACKET_IN异步队列流程
       do_send_packet_ins(ofconn, &txq);
   }

rconn_run(ofconn->rconn); //发送报文流程

/* Limit the number of iterations to avoid starving other tasks. */
for (i = 0; i < 50 && ofconn_may_recv(ofconn); i++) {
struct ofpbuf *of_msg = rconn_recv(ofconn->rconn); //接收报文流程

...

handle_openflow(ofconn, of_msg); //调用handle_openflow对openflow消息具体处理

...
}

...
}

以下将具体分析openflow消息收发流程

pinsched_run根据令牌桶分配将pinqueue队列中的报文出队，然后将该该报文放入txq中

pinsched_run( struct pinsched *ps, struct ovs_list *txq)
{
ovs_list_init(txq);
if (ps) {
int i;

/* Drain some packets out of the bucket if possible, but limit the
* number of iterations to allow other code to get work done too. */
for (i = 0; ps->n_queued && get_token(ps) && i < 50; i++) {
struct ofpbuf *packet = get_tx_packet(ps); //在ps中删除该报文，并且返回该报文
ovs_list_push_back(txq, &packet->list_node);
}
}
}

do_send_packet_ins最终将调用vconn_send发送报文。

rconn_run-->vconn_run-->vconn_stream_run-->stream_send-->fd_send最终发送报文。

rconn_run(struct rconn *rc)
{
int old_state;
size_t i;

if (rc->vconn) {
...

vconn_run(rc->vconn);

...
}

/*monitor连接后，复制得到的openflow消息发送回ofctl客户端
并将监测复制得到的of消息接收到相应rc->n_monitors[i]实例中*/
for (i = 0; i < rc->n_monitors; ) {
struct ofpbuf *msg;
int retval;

vconn_run(rc->monitors[i]);

/* Drain any stray message that came in on the monitor connection. */
retval = vconn_recv(rc->monitors[i], &msg);
if (!retval) {
ofpbuf_delete(msg);
} else if (retval != EAGAIN) {
close_monitor(rc, i, retval);
continue;
}
i++;
}

...
}

rconn_recv->vconn_recv->do_recv->vconn_stream_recv->vconn_stream_recv__->stream_recv

rconn_recv(struct rconn *rc)
{
struct ofpbuf *buffer = NULL;

ovs_mutex_lock(&rc->mutex);
if (rc->state & (S_ACTIVE | S_IDLE)) {
int error = vconn_recv(rc->vconn, &buffer);
if (!error) {
copy_to_monitor(rc, buffer); //将vconn收的报文复制到每一个monitor
...
}
return buffer;
}

收发包过程最终都是操作ofpbuf结构体，该结构体图如下：

本文主要参考：https://www.sdnlab.com/16144.html，加上了一点点自己的理解，如有错误，请悉心指正，谢谢。