ip_vs实现分析(2)
本文档的Copyleft归yfydz所有，使用GPL发布，可以自由拷贝，转载，转载时请保持文档的完整性，严禁用于任何商业用途。
msn: yfydz_no1@hotmail.com
来源：http://yfydz.cublog.cn
4. 模块初始化初始化函数先初始化ipvs的各种处理机制，然后将ipvs的处理函数挂接到netfilter架构中。/* net/ipv4/ipvs/ip_vs_core.c */static int __init ip_vs_init(void){ int ret;// ioctl初始化 ret = ip_vs_control_init(); if (ret < 0) { IP_VS_ERR("can't setup control.\n"); goto cleanup_nothing; }// 协议初始化 ip_vs_protocol_init();// 应用层辅助协议初始化 ret = ip_vs_app_init(); if (ret < 0) { IP_VS_ERR("can't setup application helper.\n"); goto cleanup_protocol; }// ipvs连接初始化 ret = ip_vs_conn_init(); if (ret < 0) { IP_VS_ERR("can't setup connection table.\n"); goto cleanup_app; }// 下面分别挂接各个处理点到netfilter架构中 ret = nf_register_hook(&ip_vs_in_ops); if (ret < 0) { IP_VS_ERR("can't register in hook.\n"); goto cleanup_conn; } ret = nf_register_hook(&ip_vs_out_ops); if (ret < 0) { IP_VS_ERR("can't register out hook.\n"); goto cleanup_inops; } ret = nf_register_hook(&ip_vs_post_routing_ops); if (ret < 0) { IP_VS_ERR("can't register post_routing hook.\n"); goto cleanup_outops; } ret = nf_register_hook(&ip_vs_forward_icmp_ops); if (ret < 0) { IP_VS_ERR("can't register forward_icmp hook.\n"); goto cleanup_postroutingops; } IP_VS_INFO("ipvs loaded.\n"); return ret;// 以下是如果初始化出现失败时依次进行释放 cleanup_postroutingops: nf_unregister_hook(&ip_vs_post_routing_ops); cleanup_outops: nf_unregister_hook(&ip_vs_out_ops); cleanup_inops: nf_unregister_hook(&ip_vs_in_ops); cleanup_conn: ip_vs_conn_cleanup(); cleanup_app: ip_vs_app_cleanup(); cleanup_protocol: ip_vs_protocol_cleanup(); ip_vs_control_cleanup(); cleanup_nothing: return ret;}4.1 ip_vs_control_init/* net/ipv4/ipvs/ip_vs_ctl.c */int ip_vs_control_init(void){ int ret; int idx; EnterFunction(2);// 登记ipvs的sockopt控制,这样用户空间可通过setsockopt函数来和ipvs进行通信 ret = nf_register_sockopt(&ip_vs_sockopts); if (ret) { IP_VS_ERR("cannot register sockopt.\n"); return ret; }// 建立/proc/net/ip_vs和/proc/net/ip_vs_stats只读项 proc_net_fops_create("ip_vs", 0, &ip_vs_info_fops); proc_net_fops_create("ip_vs_stats",0, &ip_vs_stats_fops);// 建立/proc/sys/net/ipv4/vs目录下的各可读写控制参数 sysctl_header = register_sysctl_table(vs_root_table, 0);// 初始化各种双向链表// svc_table是根据协议地址端口等信息进行服务结构struct ip_vs_service查找的HASH表// svc_fwm_table是根据数据包的nfmark信息进行服务结构struct ip_vs_service查找的HASH表 /* Initialize ip_vs_svc_table, ip_vs_svc_fwm_table, ip_vs_rtable */ for(idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { INIT_LIST_HEAD(&ip_vs_svc_table[idx]); INIT_LIST_HEAD(&ip_vs_svc_fwm_table[idx]); }// rtable是目的结构struct ip_vs_dest的HASH链表 for(idx = 0; idx < IP_VS_RTAB_SIZE; idx++) { INIT_LIST_HEAD(&ip_vs_rtable[idx]); }// ipvs统计信息 memset(&ip_vs_stats, 0, sizeof(ip_vs_stats));// 统计锁 spin_lock_init(&ip_vs_stats.lock);// 对当前统计信息建立一个预估器,可用于计算服务器的性能参数 ip_vs_new_estimator(&ip_vs_stats); /* Hook the defense timer */// 挂一个定时操作，根据系统当前负载情况定时调整系统参数 schedule_delayed_work(&defense_work, DEFENSE_TIMER_PERIOD); LeaveFunction(2); return 0;}4.2 ip_vs_protocol_init/* net/ipv4/ipvs/ip_vs_proto.c */int ip_vs_protocol_init(void){// 挂接ipvs能进行均衡处理的各种协议,目前支持TCP/UDP/AH/ESP// 最好还要增加GRE,在PPTP服务器中使用 char protocols[64];#define REGISTER_PROTOCOL(p) \ do { \ register_ip_vs_protocol(p); \ strcat(protocols, ", "); \ strcat(protocols, (p)->name); \ } while (0)// 0,1字符是给", "预留的 protocols[0] = '\0'; protocols[2] = '\0';// 登记各种协议#ifdef CONFIG_IP_VS_PROTO_TCP REGISTER_PROTOCOL(&ip_vs_protocol_tcp);#endif#ifdef CONFIG_IP_VS_PROTO_UDP REGISTER_PROTOCOL(&ip_vs_protocol_udp);#endif#ifdef CONFIG_IP_VS_PROTO_AH REGISTER_PROTOCOL(&ip_vs_protocol_ah);#endif#ifdef CONFIG_IP_VS_PROTO_ESP REGISTER_PROTOCOL(&ip_vs_protocol_esp);#endif// 第0,1字符分别为逗号','和空格' ',从第2字符起才是真正数据串 IP_VS_INFO("Registered protocols (%s)\n", &protocols[2]); return 0;}register_ip_vs_protocol()函数就是把ip_vs_protocol结构挂接到协议HASH表中，不过其实没几个协议，没必要用HASH，直接数组就行了，Linux内核中缺省好象也只支持32种IP协议。/* * register an ipvs protocol */static int register_ip_vs_protocol(struct ip_vs_protocol *pp){ unsigned hash = IP_VS_PROTO_HASH(pp->protocol);// 把新协议节点挂接到HASH链表头 pp->next = ip_vs_proto_table[hash]; ip_vs_proto_table[hash] = pp;// 调用该协议的初始化函数 if (pp->init != NULL) pp->init(pp); return 0;}4.3 ip_vs_app_initIPVS应用初始化/* net/ipv4/ipvs/ip_vs_app.c */int ip_vs_app_init(void){ /* we will replace it with proc_net_ipvs_create() soon */// 该函数就是建立一个/proc/net/ip_vs_app项 proc_net_fops_create("ip_vs_app", 0, &ip_vs_app_fops); return 0;}4.4 ip_vs_conn_initIPVS连接初始化/* net/ipv4/ipvs/ip_vs_conn.c */int ip_vs_conn_init(void){ int idx; /* * Allocate the connection hash table and initialize its list heads */// ipvs连接HASH表 ip_vs_conn_tab = vmalloc(IP_VS_CONN_TAB_SIZE*sizeof(struct list_head)); if (!ip_vs_conn_tab) return -ENOMEM; /* Allocate ip_vs_conn slab cache */// ipvs连接cache,由于使用cache在内存块释放时并不真正释放，而是cache起来，// 因此重新分配时速度更快 ip_vs_conn_cachep = kmem_cache_create("ip_vs_conn", sizeof(struct ip_vs_conn), 0, SLAB_HWCACHE_ALIGN, NULL, NULL); if (!ip_vs_conn_cachep) { vfree(ip_vs_conn_tab); return -ENOMEM; } IP_VS_INFO("Connection hash table configured " "(size=%d, memory=%ldKbytes)\n", IP_VS_CONN_TAB_SIZE, (long)(IP_VS_CONN_TAB_SIZE*sizeof(struct list_head))/1024); IP_VS_DBG(0, "Each connection entry needs %Zd bytes at least\n", sizeof(struct ip_vs_conn));// 初始化各HASH链表头 for (idx = 0; idx < IP_VS_CONN_TAB_SIZE; idx++) { INIT_LIST_HEAD(&ip_vs_conn_tab[idx]); }// 初始化各读写锁 for (idx = 0; idx < CT_LOCKARRAY_SIZE; idx++) { rwlock_init(&__ip_vs_conntbl_lock_array[idx].l); }// 建立/proc/net/ip_vs_conn项 proc_net_fops_create("ip_vs_conn", 0, &ip_vs_conn_fops); /* calculate the random value for connection hash */// 初始随机数 get_random_bytes(&ip_vs_conn_rnd, sizeof(ip_vs_conn_rnd)); return 0;}4.5 netfilter挂接点nf_hook_ops分别在FORWARD点挂2个, INPUT点和POST_ROUTING点各挂一个/* net/ipv4/ipvs/ip_vs_core.c */4.5.1 ip_vs_in_ops/* After packet filtering, forward packet through VS/DR, VS/TUN, or VS/NAT(change destination), so that filtering rules can be applied to IPVS. */static struct nf_hook_ops ip_vs_in_ops = { .hook = ip_vs_in, .owner = THIS_MODULE, .pf = PF_INET,// INPUT点 .hooknum = NF_IP_LOCAL_IN,// 此优先级低于filter .priority = 100,};ip_vs_in()这个函数对进入本机的包进行处理./* net/ipv4/ipvs/ip_vs_core.c *//* * Check if it's for virtual services, look it up, * and send it on its way... */static unsigned intip_vs_in(unsigned int hooknum, struct sk_buff **pskb, const struct net_device *in, const struct net_device *out, int (*okfn)(struct sk_buff *)){ struct sk_buff *skb = *pskb; struct iphdr *iph; struct ip_vs_protocol *pp; struct ip_vs_conn *cp; int ret, restart; int ihl; /* * Big tappo: only PACKET_HOST (neither loopback nor mcasts) * ... don't know why 1st test DOES NOT include 2nd (?) */ if (unlikely(skb->pkt_type != PACKET_HOST || skb->dev == &loopback_dev || skb->sk)) {// input不处理目的非本机的包 IP_VS_DBG(12, "packet type=%d proto=%d daddr=%d.%d.%d.%d ignored\n", skb->pkt_type, skb->nh.iph->protocol, NIPQUAD(skb->nh.iph->daddr)); return NF_ACCEPT; } iph = skb->nh.iph; if (unlikely(iph->protocol == IPPROTO_ICMP)) {// 如果是ICMP，可能是指示连接错误的ICMP信息，调用ip_vs_in_icmp进行检查// 是否是相关的ICMP信息 int related, verdict = ip_vs_in_icmp(pskb, &related, hooknum); if (related) return verdict;// 非相关ICMP，恢复处理流程// 但其实ipvs是不均衡ICMP信息的，后面就返回了 skb = *pskb; iph = skb->nh.iph; } /* Protocol supported? */// 获取协议支持模块，由于只支持TCP、UDP、AH和ESP，如果是ICMP，返回为NULL pp = ip_vs_proto_get(iph->protocol); if (unlikely(!pp)) return NF_ACCEPT; ihl = iph->ihl << 2; /* * Check if the packet belongs to an existing connection entry */// 找到和该skb相关的ipvs连接，类似netfilter的根据tuple查找连接，// 不过sk_buff结构中没有增加nfct那样能直接指向连接的成员// 对TCP协议来说是tcp_conn_in_get() cp = pp->conn_in_get(skb, pp, iph, ihl, 0); if (unlikely(!cp)) { int v;// 如果没有连接, 表明是新连接, 调用IPVS连接的conn_schedule调度连接分配和处理// 连接调度要根据调度算法选择一个真实目的服务器，然后建立新的IPVS连接// 对TCP协议来说是tcp_conn_schedule() if (!pp->conn_schedule(skb, pp, &v, &cp)) return v; } if (unlikely(!cp)) {// 这种情况主要是没内存空间了，IPVS没提供主动删除连接的机制 /* sorry, all this trouble for a no-hit :) */ IP_VS_DBG_PKT(12, pp, skb, 0, "packet continues traversal as normal"); return NF_ACCEPT; } IP_VS_DBG_PKT(11, pp, skb, 0, "Incoming packet"); /* Check the server status */ if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) { /* the destination server is not available */// 对于目的服务器失效的包丢弃 if (sysctl_ip_vs_expire_nodest_conn) { /* try to expire the connection immediately */ ip_vs_conn_expire_now(cp); } /* don't restart its timer, and silently drop the packet. */ __ip_vs_conn_put(cp); return NF_DROP; }// 连接信息统计 ip_vs_in_stats(cp, skb);// 进行连接状态的迁移, restart这个参数其实没用// 对TCP协议来说是调用tcp_state_transition restart = ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pp); if (cp->packet_xmit)// 将包发送出去, 具体xmit的实现在ip_vs_xmit.c中实现,// NAT模式下为 ip_vs_nat_xmit;// 通道模式下为 ip_vs_tunnel_xmit;// 直接路由模式下为: ip_vs_dr_xmit;// 本机数据为: ip_vs_null_xmit;// 旁路模式下为: ip_vs_bypass_xmit;// 函数成功时基本都返回NF_STOLEN使netfilter不再处理该包// 所以对于NAT模式，应该是不需要配置DNAT规则的，请求方向数据也不经过FORWARD链 ret = cp->packet_xmit(skb, cp, pp); /* do not touch skb anymore */ else { IP_VS_DBG_RL("warning: packet_xmit is null"); ret = NF_ACCEPT; } /* increase its packet counter and check if it is needed to be synchronized */ atomic_inc(&cp->in_pkts);// 在进行均衡器热备时将连接信息要从MASTER传递到SLAVE，使系统切换时// 连接不丢弃，但还是要有一定条件才进行同步 if ((ip_vs_sync_state & IP_VS_STATE_MASTER) &&// 同步状态类型为主机 (cp->protocol != IPPROTO_TCP || cp->state == IP_VS_TCP_S_ESTABLISHED) &&// 非TCP连接或是已经建立的连接 (atomic_read(&cp->in_pkts) % sysctl_ip_vs_sync_threshold[1] == sysctl_ip_vs_sync_threshold[0]))// 当前连接的包数为N*thres[1]+thres[0]时// 进行连接的同步 ip_vs_sync_conn(cp);// 调整连接超时,释放连接计数 ip_vs_conn_put(cp); return ret;}4.5.2 ip_vs_out_ops/* After packet filtering, change source only for VS/NAT */static struct nf_hook_ops ip_vs_out_ops = { .hook = ip_vs_out, .owner = THIS_MODULE, .pf = PF_INET,// FORWARD点 .hooknum = NF_IP_FORWARD,// 此优先级低于filter .priority = 100,};ip_vs_out()这个函数对转发包进行处理, 只用在NAT模式的均衡处理，TUNNEL和DR方式下都是直接发送了，实际处理的只是服务器返回的回应包，而客户端请求的包是不经过这里的，但如果设置了DNAT规则，数据包在PREROUTING点进行了目的地址修改，这样就不会再进入INPUT点而是直接转到FORWARD点处理，这时时针对该包的 IPVS连接是没有建立的。/* net/ipv4/ipvs/ip_vs_core.c *//* * It is hooked at the NF_IP_FORWARD chain, used only for VS/NAT. * Check if outgoing packet belongs to the established ip_vs_conn, * rewrite addresses of the packet and send it on its way... */static unsigned intip_vs_out(unsigned int hooknum, struct sk_buff **pskb, const struct net_device *in, const struct net_device *out, int (*okfn)(struct sk_buff *)){ struct sk_buff *skb = *pskb; struct iphdr *iph; struct ip_vs_protocol *pp; struct ip_vs_conn *cp; int ihl; EnterFunction(11);// 这个标志只占一位// 标志设上就是已经经过IPVS处理了，直接返回 if (skb->ipvs_property) return NF_ACCEPT; iph = skb->nh.iph; if (unlikely(iph->protocol == IPPROTO_ICMP)) {// 处理可能的连接相关ICMP错误信息,如地址端口不可达等 int related, verdict = ip_vs_out_icmp(pskb, &related); if (related) return verdict; skb = *pskb; iph = skb->nh.iph; }// 取得IPVS协议, tcp/udp/ah/esp之一 pp = ip_vs_proto_get(iph->protocol); if (unlikely(!pp)) return NF_ACCEPT; /* reassemble IP fragments */ if (unlikely(iph->frag_off & __constant_htons(IP_MF|IP_OFFSET) && !pp->dont_defrag)) {// 如果是碎片包进行重组,基本不可能,因为数据包进入netfilter时就要进行碎片重组 skb = ip_vs_gather_frags(skb, IP_DEFRAG_VS_OUT); if (!skb) return NF_STOLEN; iph = skb->nh.iph; *pskb = skb; } ihl = iph->ihl << 2; /* * Check if the packet belongs to an existing entry */// 查找IPVS连接 cp = pp->conn_out_get(skb, pp, iph, ihl, 0); if (unlikely(!cp)) {// 没找到IPVS连接,可能是请求方向的包经过DNAT过来的 if (sysctl_ip_vs_nat_icmp_send && (pp->protocol == IPPROTO_TCP || pp->protocol == IPPROTO_UDP)) { __u16 _ports[2], *pptr; pptr = skb_header_pointer(skb, ihl, sizeof(_ports), _ports); if (pptr == NULL) return NF_ACCEPT; /* Not for me */// 用源地址，源端口来查真实服务器结构，如果是请求方向是找不到的// 这种情况下数据包就不再被IPVS处理 if (ip_vs_lookup_real_service(iph->protocol, iph->saddr, pptr[0])) { /* * Notify the real server: there is no * existing entry if it is not RST * packet or not TCP packet. */ if (iph->protocol != IPPROTO_TCP || !is_tcp_reset(skb)) { icmp_send(skb,ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); return NF_DROP; } } } IP_VS_DBG_PKT(12, pp, skb, 0, "packet continues traversal as normal"); return NF_ACCEPT; }// 找到连接，该包是服务器的回应包 IP_VS_DBG_PKT(11, pp, skb, 0, "Outgoing packet");// skb数据包要求是可写的 if (!ip_vs_make_skb_writable(pskb, ihl)) goto drop; /* mangle the packet */// 修改协议部分信息，如TCP、UDP的端口 if (pp->snat_handler && !pp->snat_handler(pskb, pp, cp)) goto drop;// 修改源地址, 由于是服务器的返回包，只修改源地址 skb = *pskb; skb->nh.iph->saddr = cp->vaddr; ip_send_check(skb->nh.iph); IP_VS_DBG_PKT(10, pp, skb, 0, "After SNAT");// IPVS输出统计 ip_vs_out_stats(cp, skb); ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pp); ip_vs_conn_put(cp);// 对该包设置标志表示IPVS处理过了 skb->ipvs_property = 1; LeaveFunction(11); return NF_ACCEPT; drop: ip_vs_conn_put(cp); kfree_skb(*pskb); return NF_STOLEN;}4.5.3 ip_vs_post_routing_ops/* Before the netfilter connection tracking, exit from POST_ROUTING */static struct nf_hook_ops ip_vs_post_routing_ops = { .hook = ip_vs_post_routing, .owner = THIS_MODULE, .pf = PF_INET,// POSTROUTING点 .hooknum = NF_IP_POST_ROUTING,// 在源NAT之前进行 .priority = NF_IP_PRI_NAT_SRC-1,};ip_vs_post_routing()这个函数对最后要发出的包进行检查，这个包是经过FORWARD链的，源地址已经被IPVS 修改过了，不用再被netfilter进行修改。如果是IPVS处理过的包,直接跳出POSTROUTING点, 不再继续可能的该点的更低优先级的hook点操作，即不用进行netfilter标准的SNAT操作。/* net/ipv4/ipvs/ip_vs_core.c *//* * It is hooked before NF_IP_PRI_NAT_SRC at the NF_IP_POST_ROUTING * chain, and is used for VS/NAT. * It detects packets for VS/NAT connections and sends the packets * immediately. This can avoid that iptable_nat mangles the packets * for VS/NAT. */static unsigned int ip_vs_post_routing(unsigned int hooknum, struct sk_buff **pskb, const struct net_device *in, const struct net_device *out, int (*okfn)(struct sk_buff *)){// 如果没被IPVS处理过，继续后续hook点操作 if (!((*pskb)->ipvs_property)) return NF_ACCEPT; /* The packet was sent from IPVS, exit this chain */// NF_STOP和NF_ACCEPT的区别就是STOP就不继续后面的低优先级的hook_ops的操作了 return NF_STOP;}4.5.4 ip_vs_forward_icmp_ops/* After packet filtering (but before ip_vs_out_icmp), catch icmp destined for 0.0.0.0/0, which is for incoming IPVS connections */static struct nf_hook_ops ip_vs_forward_icmp_ops = { .hook = ip_vs_forward_icmp, .owner = THIS_MODULE, .pf = PF_INET,// FORWARD点 .hooknum = NF_IP_FORWARD,// 在ip_vs_out_ops之前进行 .priority = 99,};ip_vs_forward_icmp()这个函数对转发的ICMP包进行处理, 处理由于服务器失效而引起的网络或端口不可达的ICMP信息,其他和服务器无关的ICMP信息不处理/* net/ipv4/ipvs/ip_vs_core.c *//* * It is hooked at the NF_IP_FORWARD chain, in order to catch ICMP * related packets destined for 0.0.0.0/0. * When fwmark-based virtual service is used, such as transparent * cache cluster, TCP packets can be marked and routed to ip_vs_in, * but ICMP destined for 0.0.0.0/0 cannot not be easily marked and * sent to ip_vs_in_icmp. So, catch them at the NF_IP_FORWARD chain * and send them to ip_vs_in_icmp. */static unsigned intip_vs_forward_icmp(unsigned int hooknum, struct sk_buff **pskb, const struct net_device *in, const struct net_device *out, int (*okfn)(struct sk_buff *)){ int r; if ((*pskb)->nh.iph->protocol != IPPROTO_ICMP) return NF_ACCEPT;// 实际调用ip_vs_in_icmp()来处理数据包 return ip_vs_in_icmp(pskb, &r, hooknum);}/* * Handle ICMP messages in the outside-to-inside direction (incoming). * Find any that might be relevant, check against existing connections, * forward to the right destination host if relevant. * Currently handles error types - unreachable, quench, ttl exceeded. */static intip_vs_in_icmp(struct sk_buff **pskb, int *related, unsigned int hooknum){ struct sk_buff *skb = *pskb; struct iphdr *iph; struct icmphdr _icmph, *ic; struct iphdr _ciph, *cih; /* The ip header contained within the ICMP */ struct ip_vs_conn *cp; struct ip_vs_protocol *pp; unsigned int offset, ihl, verdict;// 这个参数指示该ICMP包是否和IPVS的连接相关 *related = 1; /* reassemble IP fragments */ if (skb->nh.iph->frag_off & __constant_htons(IP_MF|IP_OFFSET)) {// 进行碎片重组 skb = ip_vs_gather_frags(skb, hooknum == NF_IP_LOCAL_IN ? IP_DEFRAG_VS_IN : IP_DEFRAG_VS_FWD); if (!skb) return NF_STOLEN; *pskb = skb; } iph = skb->nh.iph; offset = ihl = iph->ihl * 4; ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph); if (ic == NULL) return NF_DROP; IP_VS_DBG(12, "Incoming ICMP (%d,%d) %u.%u.%u.%u->%u.%u.%u.%u\n", ic->type, ntohs(icmp_id(ic)), NIPQUAD(iph->saddr), NIPQUAD(iph->daddr)); /* * Work through seeing if this is for us. * These checks are supposed to be in an order that means easy * things are checked first to speed up processing.... however * this means that some packets will manage to get a long way * down this stack and then be rejected, but that's life. */ if ((ic->type != ICMP_DEST_UNREACH) && (ic->type != ICMP_SOURCE_QUENCH) && (ic->type != ICMP_TIME_EXCEEDED)) {// 如果不是这三种ICMP信息,则该skb与IPVS无关 *related = 0; return NF_ACCEPT; } /* Now find the contained IP header */ offset += sizeof(_icmph); cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); if (cih == NULL) return NF_ACCEPT; /* The packet looks wrong, ignore */// 找的是ICMP信息中包含的原始包中的协议,而不是ICMP pp = ip_vs_proto_get(cih->protocol); if (!pp) return NF_ACCEPT; /* Is the embedded protocol header present? */// 如果是碎片不处理直接返回 if (unlikely(cih->frag_off & __constant_htons(IP_OFFSET) && pp->dont_defrag)) return NF_ACCEPT; IP_VS_DBG_PKT(11, pp, skb, offset, "Checking incoming ICMP for"); offset += cih->ihl * 4; /* The embedded headers contain source and dest in reverse order */// 查找IPVS连接 cp = pp->conn_in_get(skb, pp, cih, offset, 1); if (!cp) return NF_ACCEPT;// 缺省的裁定结果是丢弃包 verdict = NF_DROP; /* Ensure the checksum is correct */ if (skb->ip_summed != CHECKSUM_UNNECESSARY &&// 检查一下IP头的校验和 ip_vs_checksum_complete(skb, ihl)) { /* Failed checksum! */ IP_VS_DBG(1, "Incoming ICMP: failed checksum from %d.%d.%d.%d!\n", NIPQUAD(iph->saddr)); goto out; } /* do the statistics and put it back */// 进行输入统计 ip_vs_in_stats(cp, skb);// 如果内部协议是TCP/UDP,发送偏移量要包括前4个字节: 源端口和目的端口 if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol) offset += 2 * sizeof(__u16);// 发送ICMP verdict = ip_vs_icmp_xmit(skb, cp, pp, offset); /* do not touch skb anymore */ out: __ip_vs_conn_put(cp); return verdict;} /* net/ipv4/ipvs/ip_vs_xmit.c *//* * ICMP packet transmitter * called by the ip_vs_in_icmp */intip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, int offset){ struct rtable *rt; /* Route to the other host */ int mtu; int rc; EnterFunction(10); /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be forwarded directly here, because there is no need to translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {// 如果不是NAT情况的IPVS连接, 即是TUNNEL或DR，直接调用连接的发送函数发送 if (cp->packet_xmit) rc = cp->packet_xmit(skb, cp, pp); else rc = NF_ACCEPT; /* do not touch skb anymore */ atomic_inc(&cp->in_pkts); goto out; } /* * mangle and send the packet here (only for VS/NAT) */// 查找路由 if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(skb->nh.iph->tos)))) goto tx_error_icmp; /* MTU checking */ mtu = dst_mtu(&rt->u.dst); if ((skb->len > mtu) && (skb->nh.iph->frag_off&__constant_htons(IP_DF))) {// 数据包过长超过MTU,但又是不允许分片的,发送ICMP出错包 ip_rt_put(rt); icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu)); IP_VS_DBG_RL("ip_vs_in_icmp(): frag needed\n"); goto tx_error; } /* copy-on-write the packet before mangling it */// 让skb可写 if (!ip_vs_make_skb_writable(&skb, offset)) goto tx_error_put;// skb留出足够的硬件头空间 if (skb_cow(skb, rt->u.dst.dev->hard_header_len)) goto tx_error_put; /* drop the old route when skb is not shared */ dst_release(skb->dst); skb->dst = &rt->u.dst;// 修改ICMP包 ip_vs_nat_icmp(skb, pp, cp, 0); /* Another hack: avoid icmp_send in ip_fragment */ skb->local_df = 1;// 将该包用OUTPUT点的hook_ops进行处理 IP_VS_XMIT(skb, rt);// NF_STOLEN表示该skb不用返回到正常的IP栈了 rc = NF_STOLEN; goto out; tx_error_icmp: dst_link_failure(skb); tx_error: dev_kfree_skb(skb); rc = NF_STOLEN; out: LeaveFunction(10); return rc; tx_error_put: ip_rt_put(rt); goto tx_error;}......待续......发表于： 2006-12-18，修改于： 2006-12-18 09:03，已浏览2486次，有评论6条 推荐 投诉网友： wujix123 时间：2007-04-04 16:25:58 IP地址：61.186.170.★yfydz是否可以详细的说明一下pp->conn_schedule(skb, pp, &v, &cp)如何转向tcp_conn_schedule(skb, pp, &v, &cp)的?网友： yfydz 时间：2007-04-05 09:31:09 IP地址：218.247.216.★对于TCP协议, pp->conn_schedule(skb, pp, &v, &cp)就是tcp_conn_schedule(skb, pp, &v, &cp)网友： 本站网友 时间：2007-04-06 09:51:31 IP地址：221.122.54.★请问一下ip_vs_out中“修改源地址, 由于是服务器的返回包，只修改源地址”，目的地址不改吗？ 这时包的目的地址应该是负载均衡器的IP吧？？网友： knewsticker 时间：2007-04-06 09:54:24 IP地址：221.122.54.★"修改源地址, 由于是服务器的返回包，只修改源地址"？？我不太清楚，这时的目的地址应该是负载均衡器的IP吧？？网友： yfydz 时间：2007-04-09 08:55:39 IP地址：218.247.216.★这里处理的包已经是服务器发给客户端的返回包了,不是客户端到服务器的包 网友： 本站网友 时间：2007-04-09 09:31:27 IP地址：221.122.54.★我建议能在分析代码前，能用图的形式把模块间的关系表示出来，好让我们这些初学者 能有一个总体的印象：）