Continue to improve the collection module

Continue to improve the collection module

Previous through acquisition module, the big difference is not bad nearing completion.

According to the principle of the development of high cohesion and low coupling, start to write a file should be run method like fishes, so it is optimized.

start.py

from src.script import run
if __name__ == '__main__':
    run()

script.py

from lib.config.settings import settings
from src.client import Agent,SSHSalt

def run():
    #判断配置文件设置的是哪种采集方式，一共三种，第一种就是agent模式，第二种和第三种因为都需要hostname而且步骤相似，于是整合成了一个方法。
    if settings.MODE == 'agent':
        Agent().collect()
    else:
        SSHSalt().collect()

client.py

from src.plugins import Plugins_Dict
from lib.config.settings import settings

import requests

class Agent():

    def collect(self):
        #### 采集资产信息
        #这就是上一篇博客的内容了，执行每一个插件，也就是每一个设备的process方法。
        res = Plugins_Dict().execute()

        for k, v in res.items():
            print(k, v)

        #### 将采集到的信息数据发送给API端，下面两种方式发送都可以
        # requests.post(settings.API_URL, data=json.dumps(res))
        requests.post(settings.API_URL, json=res)

class SSHSalt():
    #这个方法是用来向api发送请求，获取主机名列表的，下面会有api端的代码
    def getHostname(self):
        hostnames = requests.get(settings.API_URL)

        ### hostnames 就是获取的主机名列表
        # return ['c1.com', 'c2.com']
        return hostnames ### ['c1.com', 'c2.com']

    def task(self, hostname):
        res = Plugins_Dict(hostname=hostname).execute()
        #每次的执行结果都发送给api
        requests.post(settings.API, json=res)

    def collect(self):
        #获取了主机名列表，也就是域名
        hostnames = self.getHostname()
        #因为需要连接服务器，会非常耗时，所以用多线程，用线程池来实现。
        from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
        p = ThreadPoolExecutor(10)

        for hostname in hostnames:

            p.submit(self.task, hostname)

api end, temporary way, did not write.

from django.shortcuts import render,HttpResponse

# Create your views here.

def asset(request):

    if request.method == 'POST':
        print(request.body)

        return HttpResponse('okokok')
    else:

        #### 从数据库中获取服务器的主机名

        #### 最终将数据库中存的主机名以列表的格式返回
        return HttpResponse(['c1.com', 'c2.com'])

2. Data table design

2.1 server app design

• myapi: responsible for receiving client data, and then analyze client data
• backend: app management background
• respositoy: responsible for managing the data model

2.2. Data table design

Comply with the principles of:

1. Client acquisition field, and a field of the table to be consistent, convenient subsequent data directly stored database
2. Field name of the client, in the data table is to have the

Relational tables and tables:

The relationship between the Products table and the table of the server:

If you think that is one to many:

A product line (glory of the King) running on 100 servers, but only to run a business king on each server

Who glory

If you think it is many to many:

A product line (glory of the King) running on 100 servers, rather than running a business on each server

King of glory

Depends on the specific circumstances of the company may be

Data table model design principles:

1. The relationship between the table and the table
2. Table fields because the client pass over the field is what we have to design these fields

from django.db import models

class UserProfile(models.Model):
    """
    用户信息
    """
    name = models.CharField(u'姓名', max_length=32)
    email = models.EmailField(u'邮箱')
    phone = models.CharField(u'座机', max_length=32)
    mobile = models.CharField(u'手机', max_length=32)
    password = models.CharField(u'密码', max_length=64)

    class Meta:
        verbose_name_plural = "用户表"

    def __str__(self):
        return self.name

class UserGroup(models.Model):
    """
    用户组
    """
    name = models.CharField(max_length=32, unique=True)
    users = models.ManyToManyField('UserProfile')

    class Meta:
        verbose_name_plural = "用户组表"

    def __str__(self):
        return self.name

class BusinessUnit(models.Model):
    """
    业务线
    """
    name = models.CharField('业务线', max_length=64, unique=True)
    contact = models.ForeignKey('UserGroup', verbose_name='业务联系人', related_name='c', on_delete=models.CASCADE)
    manager = models.ForeignKey('UserGroup', verbose_name='系统管理员', related_name='m', on_delete=models.CASCADE)

    class Meta:
        verbose_name_plural = "业务线表"

    def __str__(self):
        return self.name

class IDC(models.Model):
    """
    机房信息
    """
    name = models.CharField('机房', max_length=32)
    floor = models.IntegerField('楼层', default=1)

    class Meta:
        verbose_name_plural = "机房表"

    def __str__(self):
        return self.name

class Tag(models.Model):
    """
    资产标签
    """
    name = models.CharField('标签', max_length=32, unique=True)

    class Meta:
        verbose_name_plural = "标签表"

    def __str__(self):
        return self.name


class Server(models.Model):
    """
    服务器信息
    """
    device_type_choices = (
        (1, '服务器'),
        (2, '交换机'),
        (3, '防火墙'),
    )
    device_status_choices = (
        (1, '上架'),
        (2, '在线'),
        (3, '离线'),
        (4, '下架'),
    )

    device_type_id = models.IntegerField('服务器类型',choices=device_type_choices, default=1)
    device_status_id = models.IntegerField('服务器状态',choices=device_status_choices, default=1)

    cabinet_num = models.CharField('机柜号', max_length=30, null=True, blank=True)
    cabinet_order = models.CharField('机柜中序号', max_length=30, null=True, blank=True)

    idc = models.ForeignKey('IDC', verbose_name='IDC机房', null=True, blank=True, on_delete=models.CASCADE)
    business_unit = models.ForeignKey('BusinessUnit', verbose_name='属于的业务线', null=True, blank=True, on_delete=models.CASCADE)

    tag = models.ManyToManyField('Tag')

    hostname = models.CharField('主机名',max_length=128, unique=True)
    sn = models.CharField('SN号', max_length=64, db_index=True)
    manufacturer = models.CharField(verbose_name='制造商', max_length=64, null=True, blank=True)
    model = models.CharField('型号', max_length=64, null=True, blank=True)

    manage_ip = models.GenericIPAddressField('管理IP', null=True, blank=True)

    os_platform = models.CharField('系统', max_length=16, null=True, blank=True)
    os_version = models.CharField('系统版本', max_length=16, null=True, blank=True)

    cpu_count = models.IntegerField('CPU个数', null=True, blank=True)
    cpu_physical_count = models.IntegerField('CPU物理个数', null=True, blank=True)
    cpu_model = models.CharField('CPU型号', max_length=128, null=True, blank=True)

    create_at = models.DateTimeField(auto_now_add=True, blank=True)

    class Meta:
        verbose_name_plural = "服务器表"

    def __str__(self):
        return self.hostname


class Disk(models.Model):
    """
    硬盘信息
    """
    slot = models.CharField('插槽位', max_length=8)
    model = models.CharField('磁盘型号', max_length=32)
    capacity = models.CharField('磁盘容量GB', max_length=32)
    pd_type = models.CharField('磁盘类型', max_length=32)
    server_obj = models.ForeignKey('Server',related_name='disk', on_delete=models.CASCADE)

    class Meta:
        verbose_name_plural = "硬盘表"

    def __str__(self):
        return self.slot


class NIC(models.Model):
    """
    网卡信息
    """
    name = models.CharField('网卡名称', max_length=128)
    hwaddr = models.CharField('网卡mac地址', max_length=64)
    netmask = models.CharField(max_length=64)
    ipaddrs = models.CharField('ip地址', max_length=256)
    up = models.BooleanField(default=False)
    server_obj = models.ForeignKey('Server',related_name='nic', on_delete=models.CASCADE)


    class Meta:
        verbose_name_plural = "网卡表"

    def __str__(self):
        return self.name


class Memory(models.Model):
    """
    内存信息
    """
    slot = models.CharField('插槽位', max_length=32)
    manufacturer = models.CharField('制造商', max_length=32, null=True, blank=True)
    model = models.CharField('型号', max_length=64)
    capacity = models.FloatField('容量', null=True, blank=True)
    sn = models.CharField('内存SN号', max_length=64, null=True, blank=True)
    speed = models.CharField('速度', max_length=16, null=True, blank=True)

    server_obj = models.ForeignKey('Server',related_name='memory', on_delete=models.CASCADE)


    class Meta:
        verbose_name_plural = "内存表"

    def __str__(self):
        return self.slot

Note: The table diagrams

api analyze data warehousing

Mainly in the hard disk data storage, for example

from django.shortcuts import render,HttpResponse

# Create your views here.
import json
from respositoy import models

def asset(request):

    if request.method == 'POST':
        res = json.loads(request.body)
        print(res)

        #### 1.获取post过来的主机名，判断主机是否合法
        status = res['basic']['status']
        if status != 10000:
            return HttpResponse('采集出错')

        hostname = res['basic']['data']['hostname']

        server_obj = models.Server.objects.filter(hostname=hostname).first()

        if not server_obj:
            return HttpResponse('资产未录入！')

        ##### 2.判断一下硬盘数据采集是否成功
        code = res['disk']['status']
        if code != 10000:
            models.ErrorLog.objects.create(asset_obj=server_obj, content=res['disk']['data'], title='采集硬盘数据出错')


        ##### 3.保存磁盘的数据入库

        new_disk_data = res['disk']['data']

        ### 新的slot集合
        new_slot = list(new_disk_data.keys())

        ### 老的slot集合
        old_disk_data = models.Disk.objects.filter(server_obj=server_obj).all()
        old_slot = []
        for obj in old_disk_data:
            old_slot.append(obj.slot)

        add_slot = set(new_slot).difference(set(old_slot))  #### [0,1,2,3,4,5]

        if add_slot:
            for slot in add_slot:
                #### {'slot': '0', 'pd_type': 'SAS', 'capacity': '279.396', 'model': 'SEAGATE ST300MM0006     LS08S0K2B5NV'},
                disk_info = res['disk']['data'][slot]
                disk_info['server_obj'] = server_obj

                models.Disk.objects.create(**disk_info)




        return HttpResponse('okokok')
    else:

        #### 从数据库中获取服务器的主机名

        #### 最终将数据库中存的主机名以列表的格式返回
        return HttpResponse(['c1.com', 'c2.com'])