Go develops a multi-cloud asset management platform cmdb
Code warehouse github.com/yunixiangfeng/gocmdb
Cloud host management
Host resource monitoring
Development Process
bee new gocmdb/server
cd gocmdb/server
go mod tidy
go get -u github.com/beego/beego/v2
go get -u "github.com/astaxie/beego/orm"
go get -u "github.com/go-sql-driver/mysql"
bee run
project structure
Modify the configuration file
D:\gocmdb\server\conf\app.conf
appname=CMDB
runmode=${RUNMODE||dev}
sessionon=true
sessionprovider=file
sessionproviderconfig=temp/session
sessionname=sid
enablexsrf=true
xsrfexpire=3600
xsrfkey=ac2e5a098492610c97ccd28ffb621014
login=AuthController.Login
home=TestController.Test
include "db.conf"
[dev]
httpport=8888
[prod]
httpport=80D:\gocmdb\server\conf\db.conf
dsn=root:1234@tcp(192.168.204.130:3306)/gocmdb?charset=utf8mb4&loc=Local&parseTime=True
Write startup file
D:\gocmdb\server\web.go
package main
import (
"flag"
"fmt"
"os"
"github.com/astaxie/beego"
"github.com/astaxie/beego/orm"
_ "github.com/go-sql-driver/mysql"
"github.com/yunixiangfeng/gocmdb/server/models"
"github.com/yunixiangfeng/gocmdb/server/utils"
_ "github.com/yunixiangfeng/gocmdb/server/routers"
)
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
init := flag.Bool("init", false, "init server")
syncdb := flag.Bool("syncdb", false, "sync db")
force := flag.Bool("force", false, "force sync db(drop table)")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: web -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/web.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
// 根据参数选择执行流程
switch {
case *init:
orm.RunSyncdb("default", *force, *verbose)
ormer := orm.NewOrm()
admin := &models.User{Name: "admin", IsSuperman: true}
if err := ormer.Read(admin, "Name"); err == orm.ErrNoRows {
password := utils.RandString(6)
admin.SetPassword(password)
if _, err := ormer.Insert(admin); err == nil {
beego.Informational("初始化admin成功, 默认密码:", password)
} else {
beego.Error("初始化用户失败, 错误:", err)
}
} else {
beego.Informational("admin用户已存在, 跳过")
}
case *syncdb:
orm.RunSyncdb("default", *force, *verbose)
beego.Informational("同步数据库")
default:
beego.Run()
}
}
log in
User/Token model definition
landing page load
Submit username/password login verification
Verification result processing
base controller
BaseController base controller, used to override the default template (default: controllerName/actionName.tpl)
D:\Workspace\Go\src\gocmdb\server\controllers\base\base.go
package base
import (
"github.com/astaxie/beego"
)
type BaseController struct {
beego.Controller
}
func (c *BaseController) Prepare() {
c.Data["xsrf_token"] = c.XSRFToken()
}
LoginRequiredController authentication controller, used for API call authentication (session/token)
D:\Workspace\Go\src\gocmdb\server\controllers\auth\auth.go
package auth
import (
"gocmdb/server/controllers/base"
"gocmdb/server/models"
)
type LoginRequiredController struct {
base.BaseController
User *models.User
}
func (c *LoginRequiredController) Prepare() {
c.BaseController.Prepare()
if user := DefaultManger.IsLogin(c); user == nil {
// 未登陆
DefaultManger.GoToLoginPage(c) // todo 需要修改参数
c.StopRun()
} else {
// 已登陆
c.User = user
c.Data["user"] = user
}
}
type AuthController struct {
base.BaseController
}
func (c *AuthController) Login() {
DefaultManger.Login(c)
}
func (c *AuthController) Logout() {
DefaultManger.Logout(c)
}
D:\Workspace\Go\src\gocmdb\server\controllers\auth\manager.go
package auth
import (
"github.com/astaxie/beego/context"
"gocmdb/server/models"
)
type AuthPlugin interface {
Name() string
Is(*context.Context) bool
IsLogin(*LoginRequiredController) *models.User
GoToLoginPage(*LoginRequiredController)
Login(*AuthController) bool
Logout(*AuthController)
}
type Manager struct {
plugins map[string]AuthPlugin
}
func NewManager() *Manager {
return &Manager{
plugins: map[string]AuthPlugin{},
}
}
func (m *Manager) Register(p AuthPlugin) {
m.plugins[p.Name()] = p
}
func (m *Manager) GetPlugin(c *context.Context) AuthPlugin {
for _, plugin := range m.plugins {
if plugin.Is(c) {
return plugin
}
}
return nil
}
func (m *Manager) IsLogin(c *LoginRequiredController) *models.User {
if plugin := m.GetPlugin(c.Ctx); plugin != nil {
return plugin.IsLogin(c)
}
return nil
}
func (m *Manager) GoToLoginPage(c *LoginRequiredController) {
if plugin := m.GetPlugin(c.Ctx); plugin != nil {
plugin.GoToLoginPage(c)
}
}
func (m *Manager) Login(c *AuthController) bool {
if plugin := m.GetPlugin(c.Ctx); plugin != nil {
return plugin.Login(c)
}
return false
}
func (m *Manager) Logout(c *AuthController) {
if plugin := m.GetPlugin(c.Ctx); plugin != nil {
plugin.Logout(c)
}
}
var DefaultManger = NewManager()
D:\Workspace\Go\src\gocmdb\server\controllers\auth\plugin.go
package auth
import (
"github.com/astaxie/beego/context"
"gocmdb/server/models"
"net/http"
"strings"
"github.com/beego/beego"
"github.com/beego/beego/validation"
"gocmdb/server/forms"
)
type Session struct {
}
func (s *Session) Name() string {
return "session"
}
func (s *Session) Is(c *context.Context) bool {
return c.Input.Header("Authentication") == ""
}
func (s *Session) IsLogin(c *LoginRequiredController) *models.User {
if session := c.GetSession("user"); session != nil {
if uid, ok := session.(int); ok {
return models.DefaultUserManager.GetById(uid)
}
}
return nil
}
func (s *Session) GoToLoginPage(c *LoginRequiredController) {
c.Redirect(beego.URLFor(beego.AppConfig.String("login")), http.StatusFound)
}
func (s *Session) Login(c *AuthController) bool {
form := &forms.LoginForm{}
valid := &validation.Validation{}
if c.Ctx.Input.IsPost() {
if err := c.ParseForm(form); err != nil {
valid.SetError("error", err.Error())
} else {
if ok, err := valid.Valid(form); err != nil {
valid.SetError("error", err.Error())
} else if ok {
c.SetSession("user", form.User.Id)
c.Redirect(beego.URLFor(beego.AppConfig.String("home")), http.StatusFound)
return true
}
}
}
c.TplName = "auth/login.html"
c.Data["form"] = form
c.Data["valid"] = valid
return false
}
func (s *Session) Logout(c *AuthController) {
c.DestroySession()
c.Redirect(beego.URLFor(beego.AppConfig.String("login")), http.StatusFound)
}
type Token struct {
}
func (t *Token) Name() string {
return "token"
}
func (t *Token) Is(c *context.Context) bool {
return strings.ToLower(strings.TrimSpace(c.Input.Header("Authentication"))) == "token"
}
func (t *Token) IsLogin(c *LoginRequiredController) *models.User {
accessKey := strings.TrimSpace(c.Ctx.Input.Header("AccessKey"))
secrectKey := strings.TrimSpace(c.Ctx.Input.Header("SecrectKey"))
if token := models.DefaultTokenManager.GetByKey(accessKey, secrectKey); token != nil && token.User.DeletedTime == nil {
return token.User
}
return nil
}
func (t *Token) GoToLoginPage(c *LoginRequiredController) {
c.Data["json"] = map[string]interface{}{
"code": 403,
"text": "请使用正确Token发起请求",
"result": nil,
}
c.ServeJSON()
}
func (t *Token) Login(c *AuthController) bool {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "请使用Token请求API",
"result": nil,
}
c.ServeJSON()
return false
}
func (t *Token) Logout(c *AuthController) {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "退出登陆成功",
"result": nil,
}
c.ServeJSON()
}
func init() {
DefaultManger.Register(new(Session))
DefaultManger.Register(new(Token))
}
D:\Workspace\Go\src\gocmdb\server\forms\auth.go
package forms
import (
"gocmdb/server/models"
"strings"
"github.com/beego/beego/validation"
)
type LoginForm struct {
Name string `form:"name"`
Password string `form:"password"`
User *models.User
}
func (f *LoginForm) Valid(v *validation.Validation) {
f.Name = strings.TrimSpace(f.Name)
f.Password = strings.TrimSpace(f.Password)
if f.Name == "" || f.Password == "" {
v.SetError("error", "用户名或密码错误")
} else {
if user := models.DefaultUserManager.GetByName(f.Name); user == nil || !user.ValidatePassword(f.Password) {
v.SetError("error", "用户名或密码错误")
} else if user.IsLock() {
v.SetError("error", "用户名被锁定")
} else {
f.User = user
}
}
}
D:\Workspace\Go\src\gocmdb\server\models\user.go
package models
import (
"time"
"gocmdb/server/utils"
"github.com/beego/beego/orm"
)
type User struct {
Id int `orm:"column(id);"`
Name string `orm:"column(name);size(32);"`
Password string `orm:"column(password);size(1024);"`
Gender int `orm:"column(gender);default(0);"`
Birthday *time.Time `orm:"column(birthday);null;default(null);"`
Tel string `orm:"column(tel);size(1024);"`
Email string `orm:"column(email);size(1024);"`
Addr string `orm:"column(addr);size(1024);"`
Remark string `orm:"column(remark);size(1024);"`
IsSuperman bool `orm:"column(is_superman);default(false);"`
Status int `orm:"column(status);"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;"`
UpdatedTime *time.Time `orm:"column(update_time);auto_now;"`
DeletedTime *time.Time `orm:"column(deleted_time);null;default(null);"`
Token *Token `orm:"reverse(one);"`
}
func (u *User) SetPassword(password string) {
u.Password = utils.Md5Salt(password, "")
}
func (u *User) ValidatePassword(password string) bool {
salt, _ := utils.SplitMd5Salt(u.Password)
return utils.Md5Salt(password, salt) == u.Password
}
func (u *User) IsLock() bool {
return u.Status == StatusLock
}
type UserManager struct{}
func NewUserManager() *UserManager {
return &UserManager{}
}
func (m *UserManager) GetById(id int) *User {
user := &User{}
err := orm.NewOrm().QueryTable(user).Filter("Id__exact", id).Filter("DeletedTime__isnull", true).One(user)
if err == nil {
return user
}
return nil
}
func (m *UserManager) GetByName(name string) *User {
user := &User{}
err := orm.NewOrm().QueryTable(user).Filter("Name__exact", name).Filter("DeletedTime__isnull", true).One(user)
if err == nil {
return user
}
return nil
}
type Token struct {
Id int `orm:"column(id);"`
User *User `orm:"column(user);rel(one);"`
AccessKey string `orm:"column(access_key);size(1024);"`
SecrectKey string `orm:"column(secrect_key);size(1024);"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;"`
UpdateTime *time.Time `orm:"column(updated_time);auto_now;"`
}
type TokenManager struct {
}
func NewTokenManager() *TokenManager {
return &TokenManager{}
}
func (m *TokenManager) GetByKey(accessKey, secrectKey string) *Token {
token := &Token{AccessKey: accessKey, SecrectKey: secrectKey}
ormer := orm.NewOrm()
if err := ormer.Read(token, "AccessKey", "SecrectKey"); err == nil {
ormer.LoadRelated(token, "User")
return token
}
return nil
}
var DefaultUserManager = NewUserManager()
var DefaultTokenManager = NewTokenManager()
func init() {
orm.RegisterModel(new(User), new(Token))
}
D:\Workspace\Go\src\gocmdb\server\utils\crypto.go
package utils
import (
"crypto/md5"
"fmt"
"strings"
)
func Md5Salt(text string, salt string) string {
if salt == "" {
salt = RandString(8)
}
return fmt.Sprintf("%s:%x", salt, md5.Sum([]byte(fmt.Sprintf("%s:%s", salt, text))))
}
func SplitMd5Salt(text string) (string, string) {
nodes := strings.SplitN(text, ":", 2)
if len(nodes) >= 2 {
return nodes[0], nodes[1]
} else {
return "", nodes[0]
}
}
D:\Workspace\Go\src\gocmdb\server\utils\rand.go
package utils
import (
"math/rand"
"time"
)
func RandString(length int) string {
letters := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"
count := len(letters)
chars := make([]byte, length)
for i := 0; i < length; i++ {
chars[i] = letters[rand.Int()%count]
// rand.Intn(count)
}
return string(chars)
}
func init() {
rand.Seed(time.Now().UnixNano())
}
D:\Workspace\Go\src\gocmdb\server\models\enum.go
package models
const (
StatusUnlock = 0
StatusLock = 1
)
D:\Workspace\Go\src\gocmdb\server\routers\router.go
package routers
import (
"github.com/astaxie/beego"
"gocmdb/server/controllers"
"gocmdb/server/controllers/auth"
)
func init() {
beego.AutoRouter(&auth.AuthController{})
beego.AutoRouter(&controllers.TestController{})
}
D:\Workspace\Go\src\gocmdb\server\views\auth\login.html
bee run
http://localhost:8888/auth/login
Enter the user name and password to jump to the test page
base controller
LayoutController Layout controller, used to set layout, layoutSections, memu and expand
D:\Workspace\Go\src\gocmdb\server\controllers\layout.go
package controllers
import "gocmdb/server/controllers/auth"
type LayoutController struct {
auth.LoginRequiredController
}
func (c *LayoutController) Prepare() {
c.LoginRequiredController.Prepare()
c.Layout = "layouts/base.html"
c.LayoutSections = map[string]string{
"LayoutStyle": "",
"LayoutScript": "",
}
c.Data["menu"] = ""
c.Data["expand"] = ""
}
ask
template
User Management:
Login verification, management page loading, user data loading, add/delete/modify/lock/unlock, Token view/generate
用户属性
Id
name varchar(32) 默认空字符串
password varchar(1024) 默认空字符串
gender int 默认0， 0 女 1 男
birthday date 允许为null, 默认值为null
tel varchar(32) 默认为空字符串
email varchar(64) 默认为空字符
addr varchar(512) 默认为空字符串
remark varchar(1024) 默认为空字符串
is_superuser bool 默认为false， true超级管理员，false普通管理员
status int 默认为0， 0表示正常，1表示锁定
created_time datetime 添加时初始化事件
updated_time datetime 更新时间，允许为null，默认为null
逻辑删除
deleted_time datetime 删除时间，允许为null，默认为null， null未删除，非null已删除User Management:
User management (add/delete (logic delete)/modify/check/lock/unlock)
Token management (generate/regenerate)
Login authentication (web session/api token)
D:\Workspace\Go\src\gocmdb\server\controllers\user.go
User Management Page Controller
package controllers
type UserPageController struct {
LayoutController
}
func (c *UserPageController) Index() {
c.Data["menu"] = "user_management"
c.Data["expand"] = "system_management"
c.TplName = "user_page/index.html"
c.LayoutSections["LayoutScript"] = "user_page/index.script.html"
}
add route
D:\Workspace\Go\src\gocmdb\server\routers\router.go
beego.AutoRouter(&controllers.UserPageController{})
D:\Workspace\Go\src\gocmdb\server\views\user_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\user_page\index.script.html
http://localhost:8888/userpage/index
User Management Controller
D:\Workspace\Go\src\gocmdb\server\controllers\user.go
User management (add/delete (logic delete)/modify/check/lock/unlock)
type UserController struct {
auth.LoginRequiredController
}
func (c *UserController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
// []*User, total, queryTotal
users, total, queryTotal := models.DefaultUserManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": users,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
func (c *UserController) Create() {
if c.Ctx.Input.IsPost() {
json := map[string]interface{}{
"code": 400,
"text": "提交数据错误",
}
form := &forms.UserCreateForm{}
valid := &validation.Validation{}
if err := c.ParseForm(form); err == nil {
if ok, err := valid.Valid(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else if ok {
user, err := models.DefaultUserManager.Create(form.Name, form.Password, form.Gender, form.BirthdayTime, form.Tel, form.Email, form.Addr, form.Remark)
if err == nil {
json = map[string]interface{}{
"code": 200,
"text": "创建成功",
"result": user,
}
} else {
json = map[string]interface{}{
"code": 500,
"text": "服务器错误",
}
}
} else {
json["result"] = valid.Errors
}
} else {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
}
c.Data["json"] = json
c.ServeJSON()
} else {
//get
c.TplName = "user/create.html"
}
}
func (c *UserController) Modify() {
if c.Ctx.Input.IsPost() {
json := map[string]interface{}{
"code": 400,
"text": "提交数据错误",
}
form := &forms.UserModifyForm{}
valid := &validation.Validation{}
if err := c.ParseForm(form); err == nil {
if ok, err := valid.Valid(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else if ok {
user, err := models.DefaultUserManager.Modify(form.Id, form.Name, form.Gender, form.BirthdayTime, form.Tel, form.Email, form.Addr, form.Remark)
if err == nil {
json = map[string]interface{}{
"code": 200,
"text": "更新成功",
"result": user,
}
} else {
json = map[string]interface{}{
"code": 500,
"text": "服务器错误",
}
}
} else {
json["result"] = valid.Errors
}
} else {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
}
c.Data["json"] = json
c.ServeJSON()
} else {
//get
pk, _ := c.GetInt("pk")
c.TplName = "user/modify.html"
c.Data["object"] = models.DefaultUserManager.GetById(pk)
}
}
func (c *UserController) Delete() {
pk, _ := c.GetInt("pk")
models.DefaultUserManager.DeleteById(pk)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "删除成功",
"result": nil, //可以返回删除的用户
}
c.ServeJSON()
}
func (c *UserController) Lock() {
pk, _ := c.GetInt("pk")
models.DefaultUserManager.SetStatusById(pk, 1)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "锁定成功",
"result": nil, //可以返回删除的用户
}
c.ServeJSON()
}
func (c *UserController) UnLock() {
pk, _ := c.GetInt("pk")
models.DefaultUserManager.SetStatusById(pk, 0)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "解锁成功",
"result": nil, //可以返回删除的用户
}
c.ServeJSON()
}
func (c *UserController) Password() {
if c.Ctx.Input.IsPost() {
json := map[string]interface{}{
"code": 400,
"text": "提交数据错误",
}
form := &forms.UserPasswordForm{User: c.User}
valid := &validation.Validation{}
if err := c.ParseForm(form); err == nil {
if ok, err := valid.Valid(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else if ok {
err := models.DefaultUserManager.UpdatePassword(c.User.Id, form.Password)
if err == nil {
json = map[string]interface{}{
"code": 200,
"text": "修改密码成功",
}
} else {
json = map[string]interface{}{
"code": 500,
"text": "服务器错误",
}
}
} else {
json["result"] = valid.Errors
}
} else {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
}
c.Data["json"] = json
c.ServeJSON()
} else {
c.TplName = "user/password.html"
}
}D:\Workspace\Go\src\gocmdb\server\forms\user.go
package forms
import (
"strings"
"time"
"gocmdb/server/models"
"github.com/astaxie/beego/validation"
)
type UserCreateForm struct {
Name string `form:"name"`
Password string `form:"password"`
PasswordVerify string `form:"passwordVerify"`
Gender int `form:"gender"`
Birthday string `form:"birthday"`
Tel string `form:"tel"`
Email string `form:"email"`
Addr string `form:"addr"`
Remark string `form:"remark"`
BirthdayTime *time.Time
}
func (f *UserCreateForm) Valid(v *validation.Validation) {
f.Name = strings.TrimSpace(f.Name)
f.Password = strings.TrimSpace(f.Password)
f.PasswordVerify = strings.TrimSpace(f.PasswordVerify)
f.Tel = strings.TrimSpace(f.Tel)
f.Email = strings.TrimSpace(f.Email)
f.Addr = strings.TrimSpace(f.Addr)
f.Remark = strings.TrimSpace(f.Remark)
v.AlphaDash(f.Name, "name.name").Message("用户名只能由数字、英文字母、中划线和下划线组成")
v.MinSize(f.Name, 5, "name.name").Message("用户名长度必须在%d-%d之内", 5, 32)
v.MaxSize(f.Name, 32, "name.name").Message("用户名长度必须在%d-%d之内", 5, 32)
if _, ok := v.ErrorsMap["name"]; !ok && models.DefaultUserManager.GetByName(f.Name) != nil {
v.SetError("name", "用户名已存在")
}
v.MinSize(f.Password, 6, "password.password").Message("密码长度必须在%d-%d之内", 6, 32)
v.MaxSize(f.Password, 32, "password.password").Message("密码长度必须在%d-%d之内", 6, 32)
if f.PasswordVerify != f.PasswordVerify {
v.SetError("passwordVerify", "两次输入密码不一致")
}
v.Range(f.Gender, 0, 1, "gender.gender").Message("性别选择不正确")
if birthday, err := time.Parse("2006-01-02", f.Birthday); err != nil {
v.SetError("birthday", "出生日期不正确")
} else {
f.BirthdayTime = &birthday
}
v.Phone(f.Tel, "tel.tel").Message("电话格式不正确")
v.Email(f.Email, "email.email").Message("邮箱格式不正确")
v.MaxSize(f.Addr, 512, "addr.addr").Message("住址长度必须在512个字符之内")
v.MaxSize(f.Remark, 512, "remark.remark").Message("备注长度必须在512个字符之内")
}
type UserModifyForm struct {
Id int `form:"id"`
Name string `form:"name"`
Gender int `form:"gender"`
Birthday string `form:"birthday"`
Tel string `form:"tel"`
Email string `form:"email"`
Addr string `form:"addr"`
Remark string `form:"remark"`
BirthdayTime *time.Time
}
func (f *UserModifyForm) Valid(v *validation.Validation) {
f.Name = strings.TrimSpace(f.Name)
f.Tel = strings.TrimSpace(f.Tel)
f.Email = strings.TrimSpace(f.Email)
f.Addr = strings.TrimSpace(f.Addr)
f.Remark = strings.TrimSpace(f.Remark)
if models.DefaultUserManager.GetById(f.Id) == nil {
v.SetError("error", "操作对象不存在")
return
}
v.AlphaDash(f.Name, "name.name").Message("用户名只能由数字、英文字母、中划线和下划线组成")
v.MinSize(f.Name, 5, "name.name").Message("用户名长度必须在%d-%d之内", 5, 32)
v.MaxSize(f.Name, 32, "name.name").Message("用户名长度必须在%d-%d之内", 5, 32)
if _, ok := v.ErrorsMap["name"]; !ok {
if user := models.DefaultUserManager.GetByName(f.Name); user != nil && user.Id != f.Id {
v.SetError("name", "用户名已存在")
}
}
v.Range(f.Gender, 0, 1, "gender.gender").Message("性别选择不正确")
if birthday, err := time.Parse("2006-01-02", f.Birthday); err != nil {
v.SetError("birthday", "出生日期不正确")
} else {
f.BirthdayTime = &birthday
}
v.Phone(f.Tel, "tel.tel").Message("电话格式不正确")
v.Email(f.Email, "email.email").Message("邮箱格式不正确")
v.MaxSize(f.Addr, 512, "addr.addr").Message("住址长度必须在512个字符之内")
v.MaxSize(f.Remark, 512, "remark.remark").Message("备注长度必须在512个字符之内")
}
type UserPasswordForm struct {
OldPassword string `form:"oldPassword"`
Password string `form:"password"`
PasswordVerify string `form:"passwordVerify"`
User *models.User
}
func (f *UserPasswordForm) Valid(v *validation.Validation) {
f.OldPassword = strings.TrimSpace(f.OldPassword)
f.Password = strings.TrimSpace(f.Password)
f.PasswordVerify = strings.TrimSpace(f.PasswordVerify)
if !f.User.ValidatePassword(f.OldPassword) {
v.SetError("oldPassword", "密码不正确")
}
v.MinSize(f.Password, 6, "password.password").Message("密码长度必须在%d-%d之内", 6, 32)
v.MaxSize(f.Password, 32, "password.password").Message("密码长度必须在%d-%d之内", 6, 32)
if f.PasswordVerify != f.PasswordVerify {
v.SetError("passwordVerify", "两次输入密码不一致")
}
}
D:\Workspace\Go\src\gocmdb\server\routers\router.go
beego.AutoRouter(&controllers.UserController{})
D:\Workspace\Go\src\gocmdb\server\views\user\create.html
D:\Workspace\Go\src\gocmdb\server\views\user\modify.html
D:\Workspace\Go\src\gocmdb\server\views\user\password.html
D:\Workspace\Go\src\gocmdb\server\controllers\user.go
type TokenController struct {
auth.LoginRequiredController
}
func (c *TokenController) Generate() {
if c.Ctx.Input.IsPost() {
pk, _ := c.GetInt("pk")
models.DefaultTokenManager.GenerateByUser(models.DefaultUserManager.GetById(pk))
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "生成Token成功",
"result": nil, //可以返回Token
}
c.ServeJSON()
} else {
pk, _ := c.GetInt("pk")
c.Data["object"] = models.DefaultUserManager.GetById(pk)
c.TplName = "token/index.html"
}
}
D:\Workspace\Go\src\gocmdb\server\routers\router.go
beego.AutoRouter(&controllers.TokenController{})
D:\Workspace\Go\src\gocmdb\server\views\token\index.html
User Management
A dialog
creation/editing process is completed,
and the lock/unlock/deletion process is completed
The current logged-in user cannot lock/delete/unlock himself.
The current user can only view and generate his own Token
change password
alert => sweetalert
1. Ajax request is not logged in and returns json
2. Create
multi-cloud management
Manage multiple cloud platforms
Alibaba Cloud
Tencent Cloud
aws
azure
Huawei Cloud
JD Cloud
Qingyun
Openstack
...
get virtual machine
start
stop
restart
server/cloud
plugins/aliyun
tenant
aws
manager
instance => vm
Type string
Name string
Init(addr region accessKey, secrectKey)
TestConnect() error
GetInstances() []*Instance
StartInstance(uuid) error
StopInstance(uuid) error
RebootInstance(uuid) error
configuration information
Authentication
Address
Region Configuration
Platform
Id
Name
Type
Addr
AccessKey
SecrectKey
Region
Remark
CreatedTime
DeletedTime
SyncTime
CreateUser rel,reverse
Status
VirtualMachine
Platform 1: n
UUID
Name
CPU
Memeory
OS
PrivateAddrs
PublicAddrs
Status string
VmCreatedTime
VmExpiredTime
CreatedTime
DeletedTime
UpdatedTime
Cloud platform management page
D:\Workspace\Go\src\gocmdb\server\conf\app.conf
home=UserPageController.Index
Cloud platform management page controller
D:\Workspace\Go\src\gocmdb\server\controllers\cloud.go
package controllers
type CloudPlatformPageController struct {
LayoutController
}
func (c *CloudPlatformPageController) Index() {
c.Data["expand"] = "cloud_management"
c.Data["menu"] = "cloud_platform_management"
c.TplName = "cloud_platform_page/index.html"
c.LayoutSections["LayoutScript"] = "cloud_platform_page/index.script.html"
}
D:\Workspace\Go\src\gocmdb\server\views\cloud_platform_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\cloud_platform_page\index.script.html
D:\Workspace\Go\src\gocmdb\server\routers\router.go
// 认证
beego.AutoRouter(&auth.AuthController{})
// 用户页面
beego.AutoRouter(&controllers.UserPageController{})
// 用户
beego.AutoRouter(&controllers.UserController{})
beego.AutoRouter(&controllers.TokenController{})
// 云平台页面
beego.AutoRouter(&controllers.CloudPlatformPageController{})
Cloud Platform Management Controller
D:\Workspace\Go\src\gocmdb\server\controllers\cloud.go
type CloudPlatformController struct {
auth.LoginRequiredController
}
func (c *CloudPlatformController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
result, total, queryTotal := models.DefaultCloudPlatformManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
func (c *CloudPlatformController) Create() {
if c.Ctx.Input.IsPost() {
form := &forms.CloudPlatformCreateForm{}
valid := &validation.Validation{}
json := map[string]interface{}{
"code": 400,
"text": "提交数据错误",
"result": nil,
}
if err := c.ParseForm(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else {
if ok, err := valid.Valid(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else if ok {
result, err := models.DefaultCloudPlatformManager.Create(
form.Name,
form.Type,
form.Addr,
form.Region,
form.AccessKey,
form.SecrectKey,
form.Remark,
c.User,
)
if err == nil {
json = map[string]interface{}{
"code": 200,
"text": "创建成功",
"result": result,
}
} else {
json = map[string]interface{}{
"code": 500,
"text": "创建失败, 请重试",
"result": nil,
}
}
} else {
json["result"] = valid.Errors
}
}
c.Data["json"] = json
c.ServeJSON()
} else {
c.TplName = "cloud_platform/create.html"
c.Data["types"] = cloud.DefaultManager.Plugins
}
}
func (c *CloudPlatformController) Delete() {
if c.Ctx.Input.IsPost() {
pk, _ := c.GetInt("pk")
models.DefaultCloudPlatformManager.DeleteById(pk)
}
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "删除成功",
"result": nil,
}
c.ServeJSON()
}Cloud platform data display list
Cloud platform creation create
Cloud platform modification modify
Cloud platform operations disable disable, enable enable, delete delete
D:\Workspace\Go\src\gocmdb\server\models\cloud.go
package models
import (
"time"
"github.com/astaxie/beego/orm"
)
type CloudPlatform struct {
Id int `orm:"column(id);" json:"id"`
Name string `orm:"column(name);size(64);" json:"name"`
Type string `orm:"column(type);size(32);" json:"type"`
Addr string `orm:"column(addr);size(1024);" json:"addr"`
AccessKey string `orm:"column(access_key);size(1024);" json:"-"`
SecrectKey string `orm:"column(secrect_key);size(1024);" json:"-"`
Region string `orm:"column(region);size(64);" json:"region"`
Remark string `orm:"column(remark);size(1024);" json:"remark"`
CreatedTime *time.Time `orm:"column(created_time);type(datetime);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);type(datetime);null;" json:"deleted_time"`
SyncTime *time.Time `orm:"column(sync_time);type(datetime);null;" json:"sync_time"`
User *User `orm:"column(user);rel(fk);" json:"user"`
Status int `orm:"column(status);" json:"status"`
VirtualMachines []*VirtualMachine `orm:"reverse(many);" json:"virtual_machines"`
}
func (p *CloudPlatform) IsEnable() bool {
return p.Status == 0
}
type CloudPlatformManager struct{}
func (m *CloudPlatformManager) Query(q string, start int64, length int) ([]*CloudPlatform, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&CloudPlatform{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
qtotal := total
if q != "" {
query := orm.NewCondition()
query = query.Or("name__icontains", q)
query = query.Or("addr__icontains", q)
query = query.Or("remark__icontains", q)
query = query.Or("region__icontains", q)
condition = condition.AndCond(query)
qtotal, _ = queryset.SetCond(condition).Count()
}
var result []*CloudPlatform
queryset.SetCond(condition).Limit(length).Offset(start).All(&result)
return result, total, qtotal
}
func NewCloudPlatformManager() *CloudPlatformManager {
return &CloudPlatformManager{}
}
func (m *CloudPlatformManager) GetByName(name string) *CloudPlatform {
ormer := orm.NewOrm()
// var result CloudPlatform
result := &CloudPlatform{}
err := ormer.QueryTable(&CloudPlatform{}).Filter("deleted_time__isnull", true).Filter("name__exact", name).One(result)
if err == nil {
return result
}
return nil
}
func (m *CloudPlatformManager) Create(name, typ, addr, region, accessKey, secrectKey, remark string, user *User) (*CloudPlatform, error) {
ormer := orm.NewOrm()
result := &CloudPlatform{
Name: name,
Type: typ,
Addr: addr,
Region: region,
AccessKey: accessKey,
SecrectKey: secrectKey,
Remark: remark,
User: user,
Status: 0,
}
if _, err := ormer.Insert(result); err != nil {
return nil, err
}
return result, nil
}
func (m *CloudPlatformManager) DeleteById(id int) error {
orm.NewOrm().QueryTable(&CloudPlatform{}).Filter("Id__exact", id).Update(orm.Params{"DeletedTime": time.Now()})
return nil
}
type VirtualMachine struct {
Id int `orm:"column(id)" json:"id"`
Platform *CloudPlatform `orm:"column(platform);rel(fk);" json:"platform"`
UUID string `orm:"column(uuid);size(128);" json:"uuid"`
Name string `orm:"column(name);size(64);" json:"name"`
CPU int `orm:"column(cpu);" json:"cpu"`
Mem int64 `orm:"column(mem);" json:"mem"`
OS string `orm:"column(os);size(128);" json:"os"`
PrivateAddrs string `orm:"column(private_addrs);size(1024);" json:"private_addrs"`
PublicAddrs string `orm:"column(public_addrs);size(1024);" json:"public_addrs"`
Status string `orm:"column(status);size(32);" json:"status"`
VmCreatedTime string `orm:"column(vm_created_time);" json:"vm_created_time"`
VmExpiredTime string `orm:"column(vm_expired_time);" json:"vm_expired_time"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;type(datetime);" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);type(datetime);null" json:"deleted_time"`
UpdatedTime *time.Time `orm:"column(updated_time);auto_now;type(datetime);" json:"updated_time"`
}
type VirtualMachineManager struct{}
func NewVirtualMachineManager() *VirtualMachineManager {
return &VirtualMachineManager{}
}
func (m *VirtualMachineManager) Query(q string, start int64, length int) ([]*VirtualMachine, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&VirtualMachine{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
qtotal := total
if q != "" {
query := orm.NewCondition()
query = query.Or("name__icontains", q)
query = query.Or("public_addrs__icontains", q)
query = query.Or("private_addrs__icontains", q)
query = query.Or("os__icontains", q)
condition = condition.AndCond(query)
qtotal, _ = queryset.SetCond(condition).Count()
}
var result []*VirtualMachine
queryset.SetCond(condition).Limit(length).Offset(start).All(&result)
return result, total, qtotal
}
var DefaultCloudPlatformManager = NewCloudPlatformManager()
var DefaultVirtualMachineManager = NewVirtualMachineManager()
func init() {
orm.RegisterModel(&CloudPlatform{}, new(VirtualMachine))
}
D:\Workspace\Go\src\gocmdb\server\forms\cloud.go
package forms
import (
"strings"
"github.com/astaxie/beego/validation"
"gocmdb/server/cloud"
"gocmdb/server/models"
)
type CloudPlatformCreateForm struct {
Name string `form:"name"`
Type string `form:"type"`
Addr string `form:"addr"`
AccessKey string `form:"access_key"`
SecrectKey string `form:"secrect_key"`
Region string `form:"region"`
Remark string `form:"remark"`
}
func (f *CloudPlatformCreateForm) Valid(v *validation.Validation) {
f.Name = strings.TrimSpace(f.Name)
f.Type = strings.TrimSpace(f.Type)
f.Addr = strings.TrimSpace(f.Addr)
f.AccessKey = strings.TrimSpace(f.AccessKey)
f.SecrectKey = strings.TrimSpace(f.SecrectKey)
f.Region = strings.TrimSpace(f.Region)
f.Remark = strings.TrimSpace(f.Remark)
v.AlphaDash(f.Name, "name.name").Message("名字只能由大小写英文、数字、下划线和中划线组成")
v.MinSize(f.Name, 5, "name.name").Message("名字长度必须在%d-%d之内", 5, 32)
v.MaxSize(f.Name, 32, "name.name").Message("名字长度必须在%d-%d之内", 5, 32)
if _, ok := v.ErrorsMap["name"]; !ok && models.DefaultCloudPlatformManager.GetByName(f.Name) != nil {
v.SetError("name", "名称已存在")
}
v.MinSize(f.Addr, 1, "addr.addr").Message("地址不能为空且长度必须在%d之内", 1024)
v.MaxSize(f.Addr, 1024, "addr.addr").Message("地址不能为空且长度必须在%d之内", 1024)
v.MinSize(f.Region, 1, "region.region").Message("区域不能为空且长度必须在%d之内", 64)
v.MaxSize(f.Region, 64, "region.region").Message("区域不能为空且长度必须在%d之内", 64)
v.MinSize(f.AccessKey, 1, "access_key.access_key").Message("AccessKey不能为空且长度必须在%d之内", 1024)
v.MaxSize(f.AccessKey, 1024, "access_key.access_key").Message("AccessKey不能为空不能为空且长度必须在%d之内", 1024)
v.MinSize(f.SecrectKey, 1, "secrect_key.secrect_key").Message("SecrectKey不能为空且长度必须在%d之内", 1024)
v.MaxSize(f.SecrectKey, 1024, "secrect_key.secrect_key").Message("SecrectKey不能为空且长度必须在%d之内", 1024)
v.MaxSize(f.Remark, 1024, "remark.remark").Message("备注长度必须在%d之内", 1024)
if sdk, ok := cloud.DefaultManager.Cloud(f.Type); !ok {
v.SetError("type", "类型错误")
} else if !v.HasErrors() {
sdk.Init(f.Addr, f.Region, f.AccessKey, f.SecrectKey)
if sdk.TestConnect() != nil {
v.SetError("type", "配置参数错误")
}
}
}
D:\Workspace\Go\src\gocmdb\server\controllers\cloud.go
type VirtualMachinePageController struct {
LayoutController
}
func (c *VirtualMachinePageController) Index() {
c.Data["expand"] = "cloud_management"
c.Data["menu"] = "virtual_machine_management"
c.TplName = "virtual_machine_page/index.html"
c.LayoutSections["LayoutScript"] = "virtual_machine_page/index.script.html"
}
type VirtualMachineController struct {
auth.LoginRequiredController
}
func (c *VirtualMachineController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
result, total, queryTotal := models.DefaultVirtualMachineManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}D:\Workspace\Go\src\gocmdb\server\routers\router.go
package routers
import (
"github.com/astaxie/beego"
"gocmdb/server/controllers"
"gocmdb/server/controllers/auth"
)
func init() {
// 认证
beego.AutoRouter(&auth.AuthController{})
// 用户页面
beego.AutoRouter(&controllers.UserPageController{})
// 用户
beego.AutoRouter(&controllers.UserController{})
beego.AutoRouter(&controllers.TokenController{})
// 云平台页面
beego.AutoRouter(&controllers.CloudPlatformPageController{})
// 云平台
beego.AutoRouter(&controllers.CloudPlatformController{})
// 云主机页面
beego.AutoRouter(&controllers.VirtualMachinePageController{})
// 云主机
beego.AutoRouter(&controllers.VirtualMachineController{})
}
virtual machine page
virtual machine operation
D:\Workspace\Go\src\gocmdb\server\cloud\base.go
package cloud
type Instance struct {
}
type ICloud interface {
Type() string
Name() string
Init(string, string, string, string)
TestConnect() error
GetInstance() []*Instance
StartInstance(string) error
StopInstance(string) error
RebootInstance(string) error
}
D:\Workspace\Go\src\gocmdb\server\cloud\manager.go
package cloud
type Manager struct {
Plugins map[string]ICloud
}
func NewManager() *Manager {
return &Manager{
Plugins: make(map[string]ICloud),
}
}
func (m *Manager) Register(c ICloud) {
m.Plugins[c.Type()] = c
}
func (m *Manager)Cloud(typ string) (ICloud, bool) {
cloud, ok := m.Plugins[typ]
return cloud, ok
}
var DefaultManager = NewManager()D:\Workspace\Go\src\gocmdb\server\cloud\plugins\init.go
package plugins
import (
_ "gocmdb/server/cloud/plugins/tenant"
)
Tencent Cloud Operations
D:\Workspace\Go\src\gocmdb\server\cloud\plugins\tenant\tenant.go
package tenant
import (
"fmt"
"gocmdb/server/cloud"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
cvm "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312"
)
type TenantCloud struct {
addr string
region string
accessKey string
secrectKey string
credential *common.Credential
profile *profile.ClientProfile
}
func (c *TenantCloud) Type() string {
return "tenant"
}
func (c *TenantCloud) Name() string {
return "腾讯云"
}
func (c *TenantCloud) Init(addr, region, accessKey, secrectKey string) {
c.addr = addr
c.region = region
c.accessKey = accessKey
c.secrectKey = secrectKey
c.credential = common.NewCredential(c.accessKey, c.secrectKey)
c.profile = profile.NewClientProfile()
c.profile.HttpProfile.Endpoint = c.addr
}
func (c *TenantCloud) TestConnect() error {
client, err := cvm.NewClient(c.credential, c.region, c.profile)
if err != nil {
fmt.Println(err)
return err
}
request := cvm.NewDescribeRegionsRequest()
_, err = client.DescribeRegions(request)
fmt.Println(err)
return err
}
func (c *TenantCloud) GetInstance() []*cloud.Instance {
return nil
}
func (c *TenantCloud) StartInstance(uuid string) error {
return nil
}
func (c *TenantCloud) StopInstance(uuid string) error {
return nil
}
func (c *TenantCloud) RebootInstance(uuid string) error {
return nil
}
func init() {
cloud.DefaultManager.Register(new(TenantCloud))
}
D:\Workspace\Go\src\gocmdb\server\views\cloud_platform\create.html
D:\Workspace\Go\src\gocmdb\server\views\virtual_machine_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\virtual_machine_page\index.script.html
Alibaba Cloud Operations
D:\Workspace\Go\src\gocmdb\server\cloud\plugins\aliyun\aliyun.go
package aliyun
import (
"gocmdb/server/cloud"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
)
type Aliyun struct {
addr string
region string
accessKey string
secrectKey string
}
func (c *Aliyun) Type() string {
return "aliyun"
}
func (c *Aliyun) Name() string {
return "阿里云"
}
func (c *Aliyun) Init(addr, region, accessKey, secrectKey string) {
c.addr = addr
c.region = region
c.accessKey = accessKey
c.secrectKey = secrectKey
}
func (c *Aliyun) TestConnect() error {
client, err := ecs.NewClientWithAccessKey(c.region, c.accessKey, c.secrectKey)
if err != nil {
return err
}
request := ecs.CreateDescribeRegionsRequest()
request.Scheme = "https"
_, err = client.DescribeRegions(request)
return err
}
func (c *Aliyun) GetInstance() []*cloud.Instance {
var (
offset int = 1
limit int = 100
total int = 2
rt []*cloud.Instance
)
for offset < total {
var instances []*cloud.Instance
total, instances = c.getInstanceByOffsetLimit(offset, limit)
if offset == 1 {
rt = make([]*cloud.Instance, 0, total)
}
rt = append(rt, instances...)
}
return rt
}
func (c *Aliyun) transformStatus(status string) string {
smap := map[string]string{
"Running": cloud.StatusRunning,
"Stopped": cloud.StatusStopped,
"Starting": cloud.StatusStarting,
"Stopping": cloud.StatusStopping,
}
if rt, ok := smap[status]; ok {
return rt
}
return cloud.StatusUnknow
}
func (c *Aliyun) getInstanceByOffsetLimit(offset, limit int) (int, []*cloud.Instance) {
client, err := ecs.NewClientWithAccessKey(c.region, c.accessKey, c.secrectKey)
if err != nil {
return 0, nil
}
request := ecs.CreateDescribeInstancesRequest()
request.Scheme = "https"
request.PageNumber = requests.NewInteger(offset)
request.PageSize = requests.NewInteger(100)
response, err := client.DescribeInstances(request)
if err != nil {
return 0, nil
}
total := response.TotalCount
instances := response.Instances.Instance
rt := make([]*cloud.Instance, len(instances))
for index, instance := range instances {
publicAddrs := make([]string, 0)
privateAddrs := make([]string, 0)
if "" != instance.EipAddress.IpAddress {
publicAddrs = append(publicAddrs, instance.EipAddress.IpAddress)
}
publicAddrs = append(publicAddrs, instance.PublicIpAddress.IpAddress...)
privateAddrs = append(privateAddrs, instance.InnerIpAddress.IpAddress...)
privateAddrs = append(instance.VpcAttributes.PrivateIpAddress.IpAddress)
rt[index] = &cloud.Instance{
UUID: instance.InstanceId,
Name: instance.InstanceName,
OS: instance.OSName,
Mem: int64(instance.Memory),
CPU: instance.Cpu,
PublicAddrs: publicAddrs,
PrivateAddrs: privateAddrs,
Status: c.transformStatus(instance.Status),
CreatedTime: instance.CreationTime,
ExpiredTime: instance.ExpiredTime,
}
}
return total, rt
}
func (c *Aliyun) StartInstance(uuid string) error {
client, err := ecs.NewClientWithAccessKey(c.region, c.accessKey, c.secrectKey)
if err != nil {
return err
}
request := ecs.CreateStartInstanceRequest()
request.Scheme = "https"
request.InstanceId = uuid
_, err = client.StartInstance(request)
return err
}
func (c *Aliyun) StopInstance(uuid string) error {
client, err := ecs.NewClientWithAccessKey(c.region, c.accessKey, c.secrectKey)
if err != nil {
return err
}
request := ecs.CreateStopInstanceRequest()
request.Scheme = "https"
request.InstanceId = uuid
_, err = client.StopInstance(request)
return err
}
func (c *Aliyun) RebootInstance(uuid string) error {
client, err := ecs.NewClientWithAccessKey(c.region, c.accessKey, c.secrectKey)
if err != nil {
return err
}
request := ecs.CreateRebootInstanceRequest()
request.Scheme = "https"
request.InstanceId = uuid
_, err = client.RebootInstance(request)
return err
}
func init() {
cloud.DefaultManager.Register(new(Aliyun))
}
Tencent Cloud Operations
D:\Workspace\Go\src\gocmdb\server\cloud\plugins\tenant\tenant.go
package tenant
import (
"fmt"
"gocmdb/server/cloud"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
cvm "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312"
)
type TenantCloud struct {
addr string
region string
accessKey string
secrectKey string
credential *common.Credential
profile *profile.ClientProfile
}
func (c *TenantCloud) Type() string {
return "tenant"
}
func (c *TenantCloud) Name() string {
return "腾讯云"
}
func (c *TenantCloud) Init(addr, region, accessKey, secrectKey string) {
c.addr = addr
c.region = region
c.accessKey = accessKey
c.secrectKey = secrectKey
c.credential = common.NewCredential(c.accessKey, c.secrectKey)
c.profile = profile.NewClientProfile()
c.profile.HttpProfile.Endpoint = c.addr
}
func (c *TenantCloud) TestConnect() error {
client, err := cvm.NewClient(c.credential, c.region, c.profile)
if err != nil {
fmt.Println(err)
return err
}
request := cvm.NewDescribeRegionsRequest()
_, err = client.DescribeRegions(request)
fmt.Println(err)
return err
}
func (c *TenantCloud) GetInstance() []*cloud.Instance {
var (
offset int64 = 0
limit int64 = 100
total int64 = 1
rt []*cloud.Instance
)
for offset < total {
var instances []*cloud.Instance
total, instances = c.getInstanceByOffsetLimit(offset, limit)
// 判断第一次请求初始化rt
if offset == 0 {
rt = make([]*cloud.Instance, 0, total)
}
rt = append(rt, instances...)
offset += limit
}
return rt
}
func (c *TenantCloud) transformStatus(status string) string {
smap := map[string]string{
"PENDING": cloud.StatusPending,
"LAUNCH_FAILED": cloud.StatusLaunchFailed,
"RUNNING": cloud.StatusRunning,
"STOPPED": cloud.StatusStopped,
"STARTING": cloud.StatusStarting,
"STOPPING": cloud.StatusStopping,
"REBOOTING": cloud.StatusRebooting,
"SHUTDOWN": cloud.StatusShutdown,
"TERMINATING": cloud.StatusTerminating,
}
if rt, ok := smap[status]; ok {
return rt
}
return cloud.StatusUnknow
}
func (c *TenantCloud) getInstanceByOffsetLimit(offset, limit int64) (int64, []*cloud.Instance) {
client, err := cvm.NewClient(c.credential, c.region, c.profile)
if err != nil {
// 通过log记录
return 0, nil
}
request := cvm.NewDescribeInstancesRequest()
request.Limit = &limit
request.Offset = &offset
response, err := client.DescribeInstances(request)
if err != nil {
// 通过log记录
return 0, nil
}
//总数
total := *response.Response.TotalCount
instances := response.Response.InstanceSet
rt := make([]*cloud.Instance, len(instances))
for index, instance := range instances {
publicAddrs := make([]string, len(instance.PublicIpAddresses))
privateAddrs := make([]string, len(instance.PrivateIpAddresses))
for index, addr := range instance.PublicIpAddresses {
publicAddrs[index] = *addr
}
for index, addr := range instance.PrivateIpAddresses {
privateAddrs[index] = *addr
}
rt[index] = &cloud.Instance{
UUID: *instance.InstanceId,
Name: *instance.InstanceName,
OS: *instance.OsName,
CPU: int(*instance.CPU),
Mem: *instance.Memory * 1024,
PublicAddrs: publicAddrs,
PrivateAddrs: privateAddrs,
Status: c.transformStatus(*instance.InstanceState),
CreatedTime: *instance.CreatedTime,
ExpiredTime: *instance.ExpiredTime,
}
}
return total, rt
}
func (c *TenantCloud) StartInstance(uuid string) error {
client, err := cvm.NewClient(c.credential, c.region, c.profile)
if err != nil {
return err
}
request := cvm.NewStartInstancesRequest()
request.InstanceIds = []*string{&uuid}
_, err = client.StartInstances(request)
return err
}
func (c *TenantCloud) StopInstance(uuid string) error {
client, err := cvm.NewClient(c.credential, c.region, c.profile)
if err != nil {
return err
}
request := cvm.NewStopInstancesRequest()
request.InstanceIds = []*string{&uuid}
_, err = client.StopInstances(request)
return err
}
func (c *TenantCloud) RebootInstance(uuid string) error {
client, err := cvm.NewClient(c.credential, c.region, c.profile)
if err != nil {
return err
}
request := cvm.NewRebootInstancesRequest()
request.InstanceIds = []*string{&uuid}
_, err = client.RebootInstances(request)
return err
}
func init() {
cloud.DefaultManager.Register(new(TenantCloud))
}
D:\Workspace\Go\src\gocmdb\server\cloud\plugins\init.go
package plugins
import (
_ "gocmdb/server/cloud/plugins/tenant"
_ "gocmdb/server/cloud/plugins/aliyun"
)D:\Workspace\Go\src\gocmdb\server\cloud\base.go
package cloud
const (
StatusPending = "创建中"
StatusLaunchFailed = "创建失败"
StatusRunning = "运行中"
StatusStopped = "已停止"
StatusStarting = "开机中"
StatusStopping = "停止中"
StatusRebooting = "重启中"
StatusTerminating = "销毁中"
StatusShutdown = "停止待销毁"
StatusUnknow = "未知"
)
type Instance struct {
UUID string
Name string
OS string
CPU int
Mem int64
PublicAddrs []string
PrivateAddrs []string
Status string
CreatedTime string
ExpiredTime string
}
func (i *Instance) String() string {
return i.Name
}
type ICloud interface {
Type() string
Name() string
Init(string, string, string, string)
TestConnect() error
GetInstance() []*Instance
StartInstance(string) error
StopInstance(string) error
RebootInstance(string) error
}virtual machine synchronization
The user operates the virtual machine
D:\Workspace\Go\src\gocmdb\server\controllers\cloud.go
func (c *CloudPlatformController) Create() {
if c.Ctx.Input.IsPost() {
form := &forms.CloudPlatformCreateForm{}
valid := &validation.Validation{}
json := map[string]interface{}{
"code": 400,
"text": "提交数据错误",
"result": nil,
}
if err := c.ParseForm(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else {
if ok, err := valid.Valid(form); err != nil {
valid.SetError("error", err.Error())
json["result"] = valid.Errors
} else if ok {
result, err := models.DefaultCloudPlatformManager.Create(
form.Name,
form.Type,
form.Addr,
form.Region,
form.AccessKey,
form.SecrectKey,
form.Remark,
c.User,
)
if err == nil {
json = map[string]interface{}{
"code": 200,
"text": "创建成功",
"result": result,
}
} else {
json = map[string]interface{}{
"code": 500,
"text": "创建失败, 请重试",
"result": nil,
}
}
} else {
json["result"] = valid.Errors
}
}
c.Data["json"] = json
c.ServeJSON()
} else {
c.TplName = "cloud_platform/create.html"
c.Data["types"] = cloud.DefaultManager.Plugins
}
}
func (c *CloudPlatformController) Delete() {
if c.Ctx.Input.IsPost() {
pk, _ := c.GetInt("pk")
models.DefaultCloudPlatformManager.DeleteById(pk)
}
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "删除成功",
"result": nil,
}
c.ServeJSON()
}
type VirtualMachinePageController struct {
LayoutController
}
func (c *VirtualMachinePageController) Index() {
c.Data["expand"] = "cloud_management"
c.Data["menu"] = "virtual_machine_management"
c.TplName = "virtual_machine_page/index.html"
c.LayoutSections["LayoutScript"] = "virtual_machine_page/index.script.html"
}
type VirtualMachineController struct {
auth.LoginRequiredController
}
func (c *VirtualMachineController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
result, total, queryTotal := models.DefaultVirtualMachineManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
func (c *VirtualMachineController) Start() {
pk, _ := c.GetInt("pk")
if vm := models.DefaultVirtualMachineManager.GetById(pk); vm != nil {
fmt.Println(vm, vm.Platform)
if sdk, ok := cloud.DefaultManager.Cloud(vm.Platform.Type); ok {
fmt.Println(sdk)
sdk.Init(vm.Platform.Addr, vm.Platform.Region, vm.Platform.AccessKey, vm.Platform.SecrectKey)
if nil == sdk.StartInstance(vm.UUID) {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "启动成功",
"result": nil,
}
c.ServeJSON()
}
}
}
c.Data["json"] = map[string]interface{}{
"code": 400,
"text": "启动失败",
"result": nil,
}
c.ServeJSON()
}
func (c *VirtualMachineController) Stop() {
pk, _ := c.GetInt("pk")
if vm := models.DefaultVirtualMachineManager.GetById(pk); vm != nil {
if sdk, ok := cloud.DefaultManager.Cloud(vm.Platform.Type); ok {
sdk.Init(vm.Platform.Addr, vm.Platform.Region, vm.Platform.AccessKey, vm.Platform.SecrectKey)
if nil == sdk.StopInstance(vm.UUID) {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "停止成功",
"result": nil,
}
c.ServeJSON()
}
}
}
c.Data["json"] = map[string]interface{}{
"code": 400,
"text": "停止失败",
"result": nil,
}
c.ServeJSON()
}
func (c *VirtualMachineController) Reboot() {
pk, _ := c.GetInt("pk")
if vm := models.DefaultVirtualMachineManager.GetById(pk); vm != nil {
if sdk, ok := cloud.DefaultManager.Cloud(vm.Platform.Type); ok {
sdk.Init(vm.Platform.Addr, vm.Platform.Region, vm.Platform.AccessKey, vm.Platform.SecrectKey)
if nil == sdk.RebootInstance(vm.UUID) {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "重启成功",
"result": nil,
}
c.ServeJSON()
}
}
}
c.Data["json"] = map[string]interface{}{
"code": 400,
"text": "重启失败",
"result": nil,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\cloud.go
package main
import (
"flag"
"fmt"
"os"
"time"
"github.com/astaxie/beego"
"github.com/astaxie/beego/orm"
_ "github.com/go-sql-driver/mysql"
"gocmdb/server/models"
)
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: cloud -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/cloud.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
for now := range time.Tick(10 * time.Second) {
fmt.Println(now)
platforms, _, _ := models.DefaultCloudPlatformManager.Query("", 0, 0)
for _, platform := range platforms {
if !platform.IsEnable() {
continue
}
fmt.Println(platform)
}
}
}
D:\Workspace\Go\src\gocmdb\server\models\cloud.go
package models
import (
"fmt"
"gocmdb/server/cloud"
"strings"
"time"
"github.com/astaxie/beego/orm"
)
type CloudPlatform struct {
Id int `orm:"column(id);" json:"id"`
Name string `orm:"column(name);size(64);" json:"name"`
Type string `orm:"column(type);size(32);" json:"type"`
Addr string `orm:"column(addr);size(1024);" json:"addr"`
AccessKey string `orm:"column(access_key);size(1024);" json:"-"`
SecrectKey string `orm:"column(secrect_key);size(1024);" json:"-"`
Region string `orm:"column(region);size(64);" json:"region"`
Remark string `orm:"column(remark);size(1024);" json:"remark"`
CreatedTime *time.Time `orm:"column(created_time);type(datetime);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);type(datetime);null;" json:"deleted_time"`
SyncTime *time.Time `orm:"column(sync_time);type(datetime);null;" json:"sync_time"`
User *User `orm:"column(user);rel(fk);" json:"user"`
Status int `orm:"column(status);" json:"status"`
Msg string `orm:"column(msg);size(1024)" json:"msg"`
VirtualMachines []*VirtualMachine `orm:"reverse(many);" json:"virtual_machines"`
}
func (p *CloudPlatform) IsEnable() bool {
return p.Status == 0
}
type CloudPlatformManager struct{}
func (m *CloudPlatformManager) Query(q string, start int64, length int) ([]*CloudPlatform, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&CloudPlatform{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
qtotal := total
if q != "" {
query := orm.NewCondition()
query = query.Or("name__icontains", q)
query = query.Or("addr__icontains", q)
query = query.Or("remark__icontains", q)
query = query.Or("region__icontains", q)
condition = condition.AndCond(query)
qtotal, _ = queryset.SetCond(condition).Count()
}
var result []*CloudPlatform
queryset.SetCond(condition).Limit(length).Offset(start).All(&result)
return result, total, qtotal
}
func NewCloudPlatformManager() *CloudPlatformManager {
return &CloudPlatformManager{}
}
func (m *CloudPlatformManager) GetByName(name string) *CloudPlatform {
ormer := orm.NewOrm()
// var result CloudPlatform
result := &CloudPlatform{}
err := ormer.QueryTable(&CloudPlatform{}).Filter("deleted_time__isnull", true).Filter("name__exact", name).One(result)
if err == nil {
return result
}
return nil
}
func (m *CloudPlatformManager) Create(name, typ, addr, region, accessKey, secrectKey, remark string, user *User) (*CloudPlatform, error) {
ormer := orm.NewOrm()
result := &CloudPlatform{
Name: name,
Type: typ,
Addr: addr,
Region: region,
AccessKey: accessKey,
SecrectKey: secrectKey,
Remark: remark,
User: user,
Status: 0,
}
if _, err := ormer.Insert(result); err != nil {
return nil, err
}
return result, nil
}
func (m *CloudPlatformManager) DeleteById(id int) error {
orm.NewOrm().QueryTable(&CloudPlatform{}).Filter("Id__exact", id).Update(orm.Params{"DeletedTime": time.Now()})
return nil
}
func (m *CloudPlatformManager) SyncInfo(platform *CloudPlatform, now time.Time, msg string) error {
platform.SyncTime = &now
platform.Msg = msg
_, err := orm.NewOrm().Update(platform)
return err
}
type VirtualMachine struct {
Id int `orm:"column(id)" json:"id"`
Platform *CloudPlatform `orm:"column(platform);rel(fk);" json:"platform"`
UUID string `orm:"column(uuid);size(128);" json:"uuid"`
Name string `orm:"column(name);size(64);" json:"name"`
CPU int `orm:"column(cpu);" json:"cpu"`
Mem int64 `orm:"column(mem);" json:"mem"`
OS string `orm:"column(os);size(128);" json:"os"`
PrivateAddrs string `orm:"column(private_addrs);size(1024);" json:"private_addrs"`
PublicAddrs string `orm:"column(public_addrs);size(1024);" json:"public_addrs"`
Status string `orm:"column(status);size(32);" json:"status"`
VmCreatedTime string `orm:"column(vm_created_time);" json:"vm_created_time"`
VmExpiredTime string `orm:"column(vm_expired_time);" json:"vm_expired_time"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;type(datetime);" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);type(datetime);null" json:"deleted_time"`
UpdatedTime *time.Time `orm:"column(updated_time);auto_now;type(datetime);" json:"updated_time"`
}
type VirtualMachineManager struct{}
func NewVirtualMachineManager() *VirtualMachineManager {
return &VirtualMachineManager{}
}
func (m *VirtualMachineManager) Query(q string, platform int, start int64, length int) ([]*VirtualMachine, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&VirtualMachine{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
if q != "" {
query := orm.NewCondition()
query = query.Or("name__icontains", q)
query = query.Or("public_addrs__icontains", q)
query = query.Or("private_addrs__icontains", q)
query = query.Or("os__icontains", q)
condition = condition.AndCond(query)
}
if platform > 0 {
condition = condition.And("platform__exact", platform)
}
var result []*VirtualMachine
qtotal, _ := queryset.SetCond(condition).Count()
queryset.SetCond(condition).RelatedSel().Limit(length).Offset(start).All(&result)
return result, total, qtotal
}
func (m *VirtualMachineManager) SyncInstance(instance *cloud.Instance, platform *CloudPlatform) {
ormer := orm.NewOrm()
vm := &VirtualMachine{UUID: instance.UUID, Platform: platform}
// 是否创建， id，error
if _, _, err := ormer.ReadOrCreate(vm, "UUID", "Platform"); err != nil {
fmt.Println(err)
return
}
vm.Name = instance.Name
vm.OS = instance.OS
vm.CPU = instance.CPU
vm.Mem = instance.Mem
vm.Status = instance.Status
vm.VmCreatedTime = instance.CreatedTime
vm.VmExpiredTime = instance.ExpiredTime
vm.PublicAddrs = strings.Join(instance.PublicAddrs, ",")
vm.PrivateAddrs = strings.Join(instance.PrivateAddrs, ",")
ormer.Update(vm)
}
func (m *VirtualMachineManager) SyncInstanceStatus(now time.Time, platform *CloudPlatform) {
orm.NewOrm().QueryTable(&VirtualMachine{}).Filter("Platform__exact", platform).Filter("UpdatedTime__lt", now).Update(orm.Params{"DeletedTime": now})
orm.NewOrm().QueryTable(&VirtualMachine{}).Filter("Platform__exact", platform).Filter("UpdatedTime__gte", now).Update(orm.Params{"DeletedTime": nil})
}
func (m *VirtualMachineManager) GetById(id int) *VirtualMachine {
vm := &VirtualMachine{}
if nil == orm.NewOrm().QueryTable(new(VirtualMachine)).RelatedSel().Filter("id__exact", id).Filter("DeletedTime__isnull", true).One(vm) {
return vm
}
return nil
}
func (m *VirtualMachineManager) DeleteByPlatform(platform *CloudPlatform) {
orm.NewOrm().QueryTable(&VirtualMachine{}).Filter("Platform__exact", platform).Update(orm.Params{"DeletedTime": time.Now()})
}
func (m *VirtualMachineManager) DeleteByPlatformId(platform int) {
orm.NewOrm().QueryTable(&VirtualMachine{}).Filter("Platform__exact", platform).Update(orm.Params{"DeletedTime": time.Now()})
}
var DefaultCloudPlatformManager = NewCloudPlatformManager()
var DefaultVirtualMachineManager = NewVirtualMachineManager()
func init() {
orm.RegisterModel(&CloudPlatform{}, new(VirtualMachine))
}
D:\Workspace\Go\src\gocmdb\server\controllers\cloud.go
func (c *VirtualMachineController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
platform, _ := c.GetInt("platform")
result, total, queryTotal := models.DefaultVirtualMachineManager.Query(q, platform, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\cloud.go
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: cloud -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/cloud.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
for now := range time.Tick(10 * time.Second) {
fmt.Println(now)
platforms, _, _ := models.DefaultCloudPlatformManager.Query("", 0, 0)
for _, platform := range platforms {
if !platform.IsEnable() {
continue
}
if sdk, ok := cloud.DefaultManager.Cloud(platform.Type); !ok {
fmt.Println("云平台未注册")
} else {
sdk.Init(platform.Addr, platform.Region, platform.AccessKey, platform.SecrectKey)
if err := sdk.TestConnect(); err != nil {
fmt.Println("测试链接失败:", err)
models.DefaultCloudPlatformManager.SyncInfo(platform, now, fmt.Sprintf("测试链接失败: %s", err.Error()))
} else {
for _, instance := range sdk.GetInstance() {
models.DefaultVirtualMachineManager.SyncInstance(instance, platform)
}
models.DefaultVirtualMachineManager.SyncInstanceStatus(now, platform)
models.DefaultCloudPlatformManager.SyncInfo(platform, now, "")
}
}
}
}
}
go run cloud.go
go run web.go
10 O'Clock
Tencent Cloud
a 10 points
b 10 points
=>
a
b
deleted b
11 o'clock
a 11 o'clock
Updates before 11 o'clock are not deleted
multiple cloud platforms
A.
for cloud platform
B.
for virtual machine
C.
D. Unify the status update in the current platform (platform filtering)
E. Unify the updates in the database
b B KB MB GB TB PB
/8
b
function FileSizeb(size) {
if(size < 8) {
return size.toFixed(2) + "b";
}
size /= 8;
var index = 0;
var units = ["B", "KB", "MB", "GB", "TB", "PB"];
while(size >= 1024) {
size /= 1024;
index += 1;
}
return size.toFixed(2) + units[index];
}
agent development
log library logrus
Address: github.com/sirupsen/logrus
set up
logrus.SetLevel(logrus.DebugLevel)
logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true})
logrus.SetOutput(os.Stdout)
record log
logrus.WithFields(logrus.Fields{}).Debug (msg)
logrus.WithFields(logrus.Fields{}).Info (msg)
logrus.WithFields(logrus.Fields{}).Error(msg)
uniquely identifies uuid
Address: github.com/google/uuid
Generate uuid.New().String()
stop service singal
Access to information
http client
start up
configuration
ENScommunication
heartbeat processing
Registration information processing
log processing
plugin management
Plugin initialization & startup
heartbeat plugin
registration message
registration message
resource information
resource information
log information
register
Code warehouse github.com/yunixinagfeng/gocmdb/agent
way of communication
agent
Main functions:
Down => Up (server)
heartbeat
Agent's server information
IP/Hostname/CPU/memory/hard disk
to collect data information
CPU/memory/disk usage/IO...
Traverse all loop plug-ins every second
Compare whether the next execution time of the plug-in is < the current time
should be executed
1. url?a=b&c=d
2. form: post
body encoding => file/ not file
file: multipart/form-data
application/ x-www-form-urlencode
body json
json
agent => server
request
Request line GET url? params HTTP/1.1
request header COOKIE: xxx
HOST: XX
request body
url =》 controlelr/action
api/heartbeat/123123123123/
:uuid
api/xxx/uuid/
uuid
Hostname
IP
OS
ARCH
CPU
RAM
DISK
BootTime
time="2023-06-25T11:18:51+08:00" level=debug msg="插件执行" Name=resource Result="{74d2472e61c2448f969fe99d32a8b63b 1 {\"load\":\"{\\\"load1\\\":0,\\\"load5\\\":0,\\\"load15\\\":0}\",\"cpu_percent\":3.125,\"ram_percent\":61,\"disk_percent\":\"{\\\"C:\\\":52.42117791025501,\\\"D:\\\":31.402267189477897,\\\"E:\\\":28.87792364710162,\\\"F:\\\":33.71282548371619}\"} 2023-06-25 11:18:51.2330884 +0800 CST m=+185.998250001}"
time="2023-06-25T11:18:51+08:00" level=debug msg="日志上传成功" result="map[code:200 result:<nil> text:成功]"time="2023-06-25T11:18:58+08:00" level=debug msg="插件执行" Name=heartbeat Result="{74d2472e61c2448f969fe99d32a8b63b 2023-06-25 11:18:58.8535355 +0800 CST m=+193.618716201}"
time="2023-06-25T11:18:58+08:00" level=debug msg="上传心跳信息成功" result="map[code:200 result:<nil> text:成
功]"terminal development
start: main
D:\Workspace\Go\src\gocmdb\agent\agentd.go
package main
import (
"os"
"os/signal"
"syscall"
"github.com/sirupsen/logrus"
"gocmdb/agent/config"
"gocmdb/agent/ens"
"gocmdb/agent/plugins"
_ "gocmdb/agent/plugins/init"
)
func main() {
logrus.SetLevel(logrus.DebugLevel)
gconf, err := config.NewConfig()
if err != nil {
logrus.Error("读取配置出错:", err)
os.Exit(-1)
}
defer func() {
os.Remove(gconf.PidFile)
}()
log, err := os.OpenFile(gconf.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
logrus.Error("打开日志文件出错:", err)
os.Exit(-1)
}
defer func() {
log.Close()
}()
// logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.SetFormatter(&logrus.TextFormatter{})
// logrus.SetOutput(log)
logrus.WithFields(logrus.Fields{
"PID": gconf.PID,
"UUID": gconf.UUID,
}).Info("Agent启动")
plugins.DefaultManager.Init(gconf)
ens.NewENS(gconf).Start()
plugins.DefaultManager.Start()
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
logrus.WithFields(logrus.Fields{
"PID": gconf.PID,
"UUID": gconf.UUID,
}).Info("Agent退出")
}
Configuration: config
D:\Workspace\Go\src\gocmdb\agent\config\config.go
package config
import (
"io/ioutil"
"os"
"strconv"
"strings"
"github.com/google/uuid"
)
type Config struct {
UUID string
UUIDFile string
Endpoint string
Token string
LogFile string
PID int
PidFile string
Heartbeat chan interface{}
Register chan interface{}
Log chan interface{}
}
func NewConfig() (*Config, error) {
UUIDFile := "agentd.uuid"
PidFile := "agentd.pid"
LogFile := "logs/agent.log"
UUID := ""
if cxt, err := ioutil.ReadFile(UUIDFile); err == nil {
UUID = string(cxt)
} else if os.IsNotExist(err) {
//strings.Replace(uuid.New().String(), "-", "", -1)
UUID = strings.ReplaceAll(uuid.New().String(), "-", "")
ioutil.WriteFile(UUIDFile, []byte(UUID), os.ModePerm)
} else {
return nil, err
}
PID := os.Getpid()
ioutil.WriteFile(PidFile, []byte(strconv.Itoa(PID)), os.ModePerm)
return &Config{
Endpoint: "http://localhost:8888/v1/api",
UUID: UUID,
UUIDFile: UUIDFile,
LogFile: LogFile,
PID: PID,
PidFile: PidFile,
Heartbeat: make(chan interface{}, 64),
Register: make(chan interface{}, 64),
Log: make(chan interface{}, 10240),
}, nil
}
Communication module: ens
D:\Workspace\Go\src\gocmdb\agent\ens\ens.go
package ens
import (
"fmt"
"github.com/imroc/req"
"github.com/sirupsen/logrus"
"gocmdb/agent/config"
)
type ENS struct {
conf *config.Config
}
func NewENS(conf *config.Config) *ENS {
return &ENS{conf: conf}
}
func (s *ENS) Start() {
logrus.Info("ENS 开始运行")
go func() {
endpoint := fmt.Sprintf("%s/heartbeat/%s/", s.conf.Endpoint, s.conf.UUID)
for evt := range s.conf.Heartbeat {
response, err := req.New().Post(endpoint, req.BodyJSON(evt))
if err == nil {
result := map[string]interface{}{}
response.ToJSON(&result)
logrus.WithFields(logrus.Fields{
"result": result,
}).Debug("上传心跳信息成功")
} else {
logrus.WithFields(logrus.Fields{
"error": err,
}).Error("上传心跳信息失败")
}
}
}()
go func() {
endpoint := fmt.Sprintf("%s/register/%s/", s.conf.Endpoint, s.conf.UUID)
for evt := range s.conf.Register {
response, err := req.New().Post(endpoint, req.BodyJSON(evt))
if err == nil {
result := map[string]interface{}{}
response.ToJSON(&result)
logrus.WithFields(logrus.Fields{
"result": result,
}).Debug("注册成功")
} else {
logrus.WithFields(logrus.Fields{
"error": err,
}).Error("注册失败")
}
}
}()
go func() {
endpoint := fmt.Sprintf("%s/log/%s/", s.conf.Endpoint, s.conf.UUID)
for evt := range s.conf.Log {
response, err := req.New().Post(endpoint, req.BodyJSON(evt))
if err == nil {
result := map[string]interface{}{}
response.ToJSON(&result)
logrus.WithFields(logrus.Fields{
"result": result,
}).Debug("日志上传成功")
} else {
logrus.WithFields(logrus.Fields{
"error": err,
}).Error("日志上传成功")
}
}
}()
}
D:\Workspace\Go\src\gocmdb\agent\entity\heartbeat.go
package entity
import "time"
type Heartbeat struct {
UUID string `json:"uuid"`
Time time.Time `json:"time"`
}
func NewHeartbeat(uuid string) Heartbeat {
return Heartbeat{
UUID: uuid,
Time: time.Now(),
}
}
D:\Workspace\Go\src\gocmdb\agent\entity\log.go
package entity
import (
"time"
"encoding/json"
)
const (
LOGResource = 0X0001
)
type Log struct {
UUID string `json:"uuid"`
Type int `json:"type"`
Msg string `json:"msg"`
Time time.Time `json:"time"`
}
func NewLog(uuid string, typ int, msg interface{}) Log {
bytes, _ := json.Marshal(msg)
return Log{
UUID: uuid,
Type: typ,
Msg: string(bytes),
Time: time.Now(),
}
}
D:\Workspace\Go\src\gocmdb\agent\entity\register.go
package entity
import (
"encoding/json"
"strings"
"time"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/disk"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
"github.com/shirou/gopsutil/net"
)
type Register struct {
UUID string `json:"uuid"`
Hostname string `json:"hostname"`
IP string `json:"ip"`
OS string `json:"os"`
Arch string `json:"arch"`
CPU int `json:"cpu"`
RAM int64 `json:"ram"` // MB
Disk string `json:"disk"`
BootTime time.Time `json:"boottime"`
Time time.Time `json:"time"`
}
func NewRegister(uuid string) Register {
hostInfo, _ := host.Info()
ips := []string{}
interfaceStatList, _ := net.Interfaces()
for _, interfaceStat := range interfaceStatList {
for _, addr := range interfaceStat.Addrs {
if strings.Index(addr.Addr, ":") >= 0 {
continue
}
if strings.Index(addr.Addr, "127.") == 0 {
continue
}
nodes := strings.Split(addr.Addr, "/")
ips = append(ips, nodes[0])
}
}
ip, _ := json.Marshal(ips)
cores := 0
cpuInfoList, _ := cpu.Info()
for _, cpuInfo := range cpuInfoList {
cores += int(cpuInfo.Cores)
}
memInfo, _ := mem.VirtualMemory()
partitionInfoList, _ := disk.Partitions(true)
disks := map[string]int64{}
for _, partitionInfo := range partitionInfoList {
usageInfo, err := disk.Usage(partitionInfo.Device)
if err != nil {
continue
}
disks[usageInfo.Path] = int64(usageInfo.Total / 1024 / 1024 / 1024)
}
disk, _ := json.Marshal(disks)
return Register{
UUID: uuid,
Hostname: hostInfo.Hostname,
OS: hostInfo.OS,
IP: string(ip),
Arch: "", //hostInfo.KernelArch,
CPU: cores,
RAM: int64(memInfo.Total / 1024 / 1024),
Disk: string(disk),
BootTime: time.Unix(int64(hostInfo.BootTime), 0),
Time: time.Now(),
}
}
D:\Workspace\Go\src\gocmdb\agent\entity\resource.go
package entity
import (
"encoding/json"
"time"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/disk"
"github.com/shirou/gopsutil/load"
"github.com/shirou/gopsutil/mem"
)
type Resource struct {
Load string `json:"load"`
CPUPrecent float64 `json:"cpu_percent"`
RAMPrecent float64 `json:"ram_percent"`
DiskPrecent string `json:"disk_percent"`
}
func NewResource() Resource {
loadAvgStat, _ := load.Avg()
cpuPercents, _ := cpu.Percent(time.Second, false)
memInfo, _ := mem.VirtualMemory()
disks := map[string]float64{}
partitionInfoList, _ := disk.Partitions(true)
for _, partitionInfo := range partitionInfoList {
usageInfo, err := disk.Usage(partitionInfo.Device)
if err != nil {
continue
}
disks[usageInfo.Path] = usageInfo.UsedPercent
}
load, _ := json.Marshal(loadAvgStat)
disk, _ := json.Marshal(disks)
return Resource{
Load: string(load),
CPUPrecent: cpuPercents[0],
RAMPrecent: memInfo.UsedPercent,
DiskPrecent: string(disk),
}
}
Plugins (management, cycle plugins)
D:\Workspace\Go\src\gocmdb\agent\plugins\manager.go
package plugins
import (
"gocmdb/agent/config"
"time"
"github.com/sirupsen/logrus"
)
type Manager struct {
Cycles map[string]CyclePlugin
}
func NewManager() *Manager {
return &Manager{
Cycles: make(map[string]CyclePlugin),
}
}
func (m *Manager) RegisterCycle(p CyclePlugin) {
m.Cycles[p.Name()] = p
logrus.WithFields(logrus.Fields{
"Name": p.Name(),
}).Info("插件注册")
}
func (m *Manager) Init(conf *config.Config) {
for name, plugin := range m.Cycles {
plugin.Init(conf)
logrus.WithFields(logrus.Fields{
"Name": name,
}).Info("初始化插件")
}
}
func (m *Manager) Start() {
go m.StartCycle()
}
func (m *Manager) StartCycle() {
for now := range time.Tick(time.Second) {
for name, plugin := range m.Cycles {
if now.After(plugin.NextTime()) {
if evt, err := plugin.Call(); err == nil {
logrus.WithFields(logrus.Fields{
"Name": name,
"Result": evt,
}).Debug("插件执行")
plugin.Pipline() <- evt
} else {
logrus.WithFields(logrus.Fields{
"Name": name,
"error": err,
}).Debug("插件执行失败")
}
}
}
}
}
var DefaultManager = NewManager()
D:\Workspace\Go\src\gocmdb\agent\plugins\base.go
package plugins
import (
"gocmdb/agent/config"
"time"
)
type CyclePlugin interface {
Name() string
Init(*config.Config)
NextTime() time.Time
Call() (interface{}, error)
Pipline() chan interface{}
}
D:\Workspace\Go\src\gocmdb\agent\plugins\init\cycle.go
package init
import (
"gocmdb/agent/plugins"
"gocmdb/agent/plugins/cycle"
)
func init() {
plugins.DefaultManager.RegisterCycle(&cycle.Heartbeat{})
plugins.DefaultManager.RegisterCycle(&cycle.Register{})
plugins.DefaultManager.RegisterCycle(&cycle.Resource{})
}
D:\Workspace\Go\src\gocmdb\agent\plugins\cycle\heartbeat.go
package cycle
import (
"gocmdb/agent/config"
"gocmdb/agent/entity"
"time"
)
type Heartbeat struct {
conf *config.Config
interval time.Duration
nextTime time.Time
}
func (p *Heartbeat) Name() string {
return "heartbeat"
}
func (p *Heartbeat) Init(conf *config.Config) {
p.conf = conf
p.interval = 10 * time.Second
p.nextTime = time.Now()
}
func (p *Heartbeat) NextTime() time.Time {
return p.nextTime
}
func (p *Heartbeat) Call() (interface{}, error) {
p.nextTime = p.nextTime.Add(p.interval)
return entity.NewHeartbeat(p.conf.UUID), nil
}
func (p *Heartbeat) Pipline() chan interface{} {
return p.conf.Heartbeat
}
D:\Workspace\Go\src\gocmdb\agent\plugins\cycle\register.go
package cycle
import (
"time"
"gocmdb/agent/config"
"gocmdb/agent/entity"
)
type Register struct {
conf *config.Config
interval time.Duration
nextTime time.Time
}
func (p *Register) Name() string {
return "register"
}
func (p *Register) Init(conf *config.Config) {
p.conf = conf
// p.interval = time.Hour
p.interval = time.Second * 30
p.nextTime = time.Now()
}
func (p *Register) NextTime() time.Time {
return p.nextTime
}
func (p *Register) Call() (interface{}, error) {
p.nextTime = p.nextTime.Add(p.interval)
return entity.NewRegister(p.conf.UUID), nil
}
func (p *Register) Pipline() chan interface{} {
return p.conf.Register
}D:\Workspace\Go\src\gocmdb\agent\plugins\cycle\resource.go
package cycle
import (
"gocmdb/agent/config"
"gocmdb/agent/entity"
"time"
)
type Resource struct {
conf *config.Config
interval time.Duration
nextTime time.Time
}
func (p *Resource) Name() string {
return "resource"
}
func (p *Resource) Init(conf *config.Config) {
p.conf = conf
p.interval = time.Minute
p.nextTime = time.Now()
}
func (p *Resource) NextTime() time.Time {
return p.nextTime
}
func (p *Resource) Call() (interface{}, error) {
p.nextTime = p.nextTime.Add(p.interval)
return entity.NewLog(p.conf.UUID, entity.LOGResource, entity.NewResource()), nil
}
func (p *Resource) Pipline() chan interface{} {
return p.conf.Log
}
Configuration read (viper)
viper：https://github.com/spf13/viper
Support yaml, json and other format configuration files
Use: viper.New()
viper.SetConfigName(“agent”)
viper.AddConfigPath(“.”)
viper.SetConfigType(“yaml”)
viper.ReadInConfig()
viper.GetXXX()
server side
D:\Workspace\Go\src\gocmdb\server\controllers\api\base.go
package api
import (
"github.com/astaxie/beego"
)
type BaseController struct {
beego.Controller
}
func (c *BaseController) Prepare() {
c.EnableXSRF = false
}
D:\Workspace\Go\src\gocmdb\server\controllers\api\v1\api.go
package v1
import (
"encoding/json"
"gocmdb/server/controllers/api"
"gocmdb/server/models"
)
type APIController struct {
api.BaseController
}
func (c *APIController) Heartbeat() {
models.DefaultAgentManager.Heartbeat(c.Ctx.Input.Param(":uuid"))
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "成功",
"result": nil,
}
c.ServeJSON()
}
func (c *APIController) Register() {
rt := map[string]interface{}{
"code": 400,
"text": "成功",
"result": nil,
}
agent := &models.Agent{}
if err := json.Unmarshal(c.Ctx.Input.RequestBody, agent); err == nil {
agent, created, err := models.DefaultAgentManager.CreateOrReplace(agent)
if err == nil {
rt = map[string]interface{}{
"code": 200,
"text": "成功",
"result": map[string]interface{}{
"created": created,
"agent": agent,
},
}
} else {
rt["text"] = err.Error()
}
} else {
rt["text"] = err.Error()
}
c.Data["json"] = rt
c.ServeJSON()
}
func (c *APIController) Log() {
log := &models.Log{}
if err := json.Unmarshal(c.Ctx.Input.RequestBody, log); err == nil {
models.DefaultLogManager.Create(log)
}
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "成功",
"result": nil,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\models\log.go
package models
import (
"encoding/json"
"time"
"github.com/astaxie/beego/orm"
)
const (
LOGResource = 0X0001
)
type Log struct {
UUID string `json:"uuid"`
Type int `json:"type"`
Msg string `json:"msg"`
Time *time.Time `json:"time"`
}
type LogManager struct{}
func NewLogManager() *LogManager {
return &LogManager{}
}
func (m *LogManager) Create(log *Log) {
switch log.Type {
case LOGResource:
resource := &Resource{}
if err := json.Unmarshal([]byte(log.Msg), resource); err == nil {
DefaultResourceManager.Create(log, resource)
}
}
}
type Resource struct {
Id int `orm:"column(id);" json:"id"`
UUID string `orm:"column(uuid);size(64);" json:"uuid"`
Load string `orm:"column(load);size(1024);" json:"load"`
CPUPrecent float64 `orm:"column(cpu_percent);" json:"cpu_percent"`
RAMPrecent float64 `orm:"column(ram_percent);" json:"ram_percent"`
DiskPrecent string `orm:"column(disk_percent);size(4096);" json:"disk_percent"`
Time *time.Time `orm:"column(time);" json:"time"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
}
type ResourceManager struct{}
func NewResourceManager() *ResourceManager {
return &ResourceManager{}
}
func (m *ResourceManager) Create(log *Log, resource *Resource) {
resource.UUID = log.UUID
resource.Time = log.Time
orm.NewOrm().Insert(resource)
}
var DefaultLogManager = NewLogManager()
var DefaultResourceManager = NewResourceManager()
func init() {
orm.RegisterModel(new(Resource))
}
D:\Workspace\Go\src\gocmdb\server\models\agent.go
package models
import (
"time"
"github.com/astaxie/beego/orm"
)
type Agent struct {
Id int `orm:"column(id);" json:"id"`
UUID string `orm:"column(uuid);size(64);" json:"uuid"`
Hostname string `orm:"column(hostname);size(64);" json:"hostname"`
IP string `orm:"column(ip);size(4096);" json:"ip"`
OS string `orm:"column(os);size(64);" json:"os"`
Arch string `orm:"column(arch);size(64);" json:"arch"`
CPU int `orm:"column(cpu);" json:"cpu"`
RAM int64 `orm:"column(ram);" json:"ram"` // MB
Disk string `orm:"column(disk);size(4096);" json:"disk"`
BootTime *time.Time `orm:"column(boot_time);null;" json:"boottime"`
Time *time.Time `orm:"column(time);null;" json:"time"`
HeartbeatTime *time.Time `orm:"column(heartbeat_time);null;" json:"heartbeat_time"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
IsOnline bool `orm:"-" json:"is_onlne"`
}
type AgentManager struct{}
func NewAgentManager() *AgentManager {
return &AgentManager{}
}
func (m *AgentManager) CreateOrReplace(agent *Agent) (*Agent, bool, error) {
now := time.Now()
ormer := orm.NewOrm()
orgAgent := &Agent{UUID: agent.UUID}
if created, _, err := ormer.ReadOrCreate(orgAgent, "UUID"); err != nil {
return nil, false, err
} else {
orgAgent.Hostname = agent.Hostname
orgAgent.IP = agent.IP
orgAgent.OS = agent.OS
orgAgent.CPU = agent.CPU
orgAgent.RAM = agent.RAM
orgAgent.Disk = agent.Disk
orgAgent.BootTime = agent.BootTime
orgAgent.Time = agent.Time
orgAgent.DeletedTime = nil
orgAgent.HeartbeatTime = &now
ormer.Update(orgAgent)
return orgAgent, created, nil
}
}
func (m *AgentManager) Heartbeat(uuid string) {
orm.NewOrm().QueryTable(&Agent{}).Filter("UUID__exact", uuid).Update(orm.Params{"HeartbeatTime": time.Now()})
}
var DefaultAgentManager = NewAgentManager()
func init() {
orm.RegisterModel(new(Agent))
}
D:\Workspace\Go\src\gocmdb\server\routers\router.go
v1 "github.com/imsilence/gocmdb/server/controllers/api/v1"
v1Namespace := beego.NewNamespace("/v1",
beego.NSRouter("api/heartbeat/:uuid/", &v1.APIController{}, "*:Heartbeat"),
beego.NSRouter("api/register/:uuid/", &v1.APIController{}, "*:Register"),
beego.NSRouter("api/log/:uuid/", &v1.APIController{}, "*:Log"),
)
beego.AddNamespace(v1Namespace)Request line: GET URL?a=b HTTP/1.1
request header: Host:
Cookie:
Token:
Empty line
Request body a=b&c=d
{"a":1, "b":2}
register/uuid/
1. Terminal display
query hostname/ip
Action: Delete edit: Property description
not relevant for terminal uploads
Open the page /url/ render view
table datatable
AgentPageController
LayoutController
Index()
menu = ""
expand = ""
tplName = ""
ajax list
AgentController
List
Delete
2. Log:
Query display (uuid => terminal name)
query condition: agent uuid
time [start_time, end_time]
3. Agent -v
has -v log set to debug, open req.DEBUG
no log set to info, close req.DEBUG
The company has 200 virtual machines
Development => 5 days
v1 => 10 machines testing functionality (5 days)
=> 20 machines
After 1 month,
20 runs the v1 version
Question:
url curl initiates a request
Authentication: Token
I go to develop:
I changed the server v1 on the original basis => changed the code
development terminal, recompiled and redeployed
Server v2 => add authentication on the basis of v1, v1 api v2 api
server goes online
Add Token parameters on top of the original code
to redeploy
Automatic upgrade
binary
configuration
Token: Store the Token value yourself app.conf
Agent upload Token
Header: Token: xxx
last 12 hours
13:30 - 16:00 No start
17:30
5:30 - 17:30
13:229 16:01
1 2 3 4 5
a b 0 0 c
1 2 5
a b c
1: a
2: b
5: c
1 2 3 4 5
a b 0 0 c
startTime endTime 1min
starTime starTime + 1min startTime + 2min startTime + 3min .... startTime + n min <= endTime
17 ：45：32
17:45:30
key time: record
for range []*Resource
resoruce.create_time => resource
[]*Resource
3 consecutive CPU usage > 90% alarms
1 agent1 CPU 80
2 agent2 CPU 60
3 agent3 CPU 90
4 agent1 CPU 88
5 agent2 CPU 98
6 99
7 91
In the last X time period, the usage rate exceeds Z for Y times.
select uuid, count(*) where created_Time > startTime and cpu > 90 group by uuid hanving count(*) > Y
having
alarm
created_time generation time
uuid terminal
type type (1: offline, 2: CPU, 3: memory)
description alarm description
status status (0, not processed, 1: processing, 1: processed)
dealt_time processing time
reason description of alarm
notices notification method (sms, email)
Token authentication
read configuration
Visual component echarts introduction
CPU and memory usage in the last 12/24 hours
Start timing refresh setInterval when opening monitoring dialog
Stop timing refresh clearInterval when dialog is closed
Use jQuery.get to get terminal resource usage information
Use echarts.setOption to update chart information
Terminal offline alarm
The difference between the heartbeat time and the current time is X minutes
SELECT * FROM agent where heartbeat_time < X
CPU and memory usage alarm
Alarm frequency control
Offline alarm:
A maximum of one alarm per day per terminal
CPU, memory usage
A maximum of one alarm per hour per terminal
The gomail.v2 package provided by gopkg.in supports the sending of Email
Package address: gopkg.in/gomail.v2
use:
Define the Message object (email information, including: subject, sender, recipient, content, attachment, etc.)
Define STMP server connection object
Connect to STMP server and send mail
Short message
Tencent Cloud API https://cloud.tencent.com/document/api/382/38778
agent development
agent.yaml
endpoint: http://localhost:8888/v2/api
token: abc1234567
uuidfile: agentd.uuid
pidfile: agentd.pid
logfile: logs/agent.logD:\Workspace\Go\src\gocmdb\agent\agentd.go
func main() {
logrus.SetLevel(logrus.DebugLevel)
req.Debug = true
configReader := viper.New()
configReader.SetConfigName("agent")
configReader.SetConfigType("yaml")
configReader.AddConfigPath("etc/")
err := configReader.ReadInConfig()
if err != nil {
logrus.Error("读取配置出错:", err)
os.Exit(-1)
}
gconf, err := config.NewConfig(configReader)
if err != nil {
logrus.Error("读取配置出错:", err)
os.Exit(-1)
}
defer func() {
os.Remove(gconf.PidFile)
}()
log, err := os.OpenFile(gconf.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
logrus.Error("打开日志文件出错:", err)
os.Exit(-1)
}
defer func() {
log.Close()
}()
// logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.SetFormatter(&logrus.TextFormatter{})
// logrus.SetOutput(log)
logrus.WithFields(logrus.Fields{
"PID": gconf.PID,
"UUID": gconf.UUID,
}).Info("Agent启动")
plugins.DefaultManager.Init(gconf)
ens.NewENS(gconf).Start()
plugins.DefaultManager.Start()
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
logrus.WithFields(logrus.Fields{
"PID": gconf.PID,
"UUID": gconf.UUID,
}).Info("Agent退出")
}
D:\Workspace\Go\src\gocmdb\agent\config\config.go
func NewConfig(configReader *viper.Viper) (*Config, error) {
UUIDFile := configReader.GetString("uuidfile")
if UUIDFile == "" {
UUIDFile = "agentd.uuid"
}
PidFile := configReader.GetString("pidfile")
if PidFile == "" {
PidFile = "agentd.pid"
}
LogFile := configReader.GetString("logfile")
if LogFile == "" {
LogFile = "logs/agent.log"
}
Endpoint := configReader.GetString("endpoint")
if Endpoint == "" {
Endpoint = "http://localhost:8888/v1/api"
}
Token := configReader.GetString("token")
UUID := ""
if cxt, err := ioutil.ReadFile(UUIDFile); err == nil {
UUID = string(cxt)
} else if os.IsNotExist(err) {
//strings.Replace(uuid.New().String(), "-", "", -1)
UUID = strings.ReplaceAll(uuid.New().String(), "-", "")
ioutil.WriteFile(UUIDFile, []byte(UUID), os.ModePerm)
} else {
return nil, err
}
PID := os.Getpid()
ioutil.WriteFile(PidFile, []byte(strconv.Itoa(PID)), os.ModePerm)
return &Config{
Endpoint: Endpoint,
Token: Token,
UUID: UUID,
UUIDFile: UUIDFile,
LogFile: LogFile,
PID: PID,
PidFile: PidFile,
Heartbeat: make(chan interface{}, 64),
Register: make(chan interface{}, 64),
Log: make(chan interface{}, 10240),
}, nil
}server-side development
D:\Workspace\Go\src\gocmdb\server\conf\app.conf
copyrequestbody=true
include "agent.conf"
D:\Workspace\Go\src\gocmdb\server\conf\agent.conf
[agent]
token=abc1234567
D:\Workspace\Go\src\gocmdb\server\controllers\agent.go
package controllers
import (
"strings"
"gocmdb/server/controllers/auth"
"gocmdb/server/models"
)
type AgentPageController struct {
LayoutController
}
func (c *AgentPageController) Index() {
c.Data["menu"] = "agent_management"
c.TplName = "agent_page/index.html"
c.LayoutSections["LayoutScript"] = "agent_page/index.script.html"
}
type AgentController struct {
auth.LoginRequiredController
}
func (c *AgentController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
result, total, queryTotal := models.DefaultAgentManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
func (c *AgentController) Delete() {
if c.Ctx.Input.IsPost() {
pk, _ := c.GetInt("pk")
models.DefaultAgentManager.DeleteById(pk)
}
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "删除成功",
"result": nil,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\controllers\log.go
package controllers
import (
"strings"
"gocmdb/server/controllers/auth"
"gocmdb/server/models"
)
type ResourcePageController struct {
LayoutController
}
func (c *ResourcePageController) Index() {
c.Data["menu"] = "resource"
c.Data["expand"] = "log_management"
c.TplName = "resource_page/index.html"
c.LayoutSections["LayoutScript"] = "resource_page/index.script.html"
}
type ResourceController struct {
auth.LoginRequiredController
}
func (c *ResourceController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
result, total, queryTotal := models.DefaultResourceManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
func (c *ResourceController) Trend() {
uuid := strings.TrimSpace(c.GetString("uuid"))
result := models.DefaultResourceManager.Trend(uuid)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\models\agent.go
package models
import (
"github.com/astaxie/beego/orm"
"time"
"encoding/json"
)
type Agent struct {
Id int `orm:"column(id);" json:"id"`
UUID string `orm:"column(uuid);size(64);" json:"uuid"`
Hostname string `orm:"column(hostname);size(64);" json:"hostname"`
IP string `orm:"column(ip);size(4096);" json:"ip"`
OS string `orm:"column(os);size(64);" json:"os"`
Arch string `orm:"column(arch);size(64);" json:"arch"`
CPU int `orm:"column(cpu);" json:"cpu"`
RAM int64 `orm:"column(ram);" json:"ram"` // MB
Disk string `orm:"column(disk);size(4096);" json:"disk"`
BootTime *time.Time `orm:"column(boot_time);null;" json:"boottime"`
Time *time.Time `orm:"column(time);null;" json:"time"`
HeartbeatTime *time.Time `orm:"column(heartbeat_time);null;" json:"heartbeat_time"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
IsOnline bool `orm:"-" json:"is_online"`
IPList []string `orm:"-" json:"ip_list"`
Disks map[string]float64 `orm:"-" json:"disks"`
}
func (a *Agent) Patch() {
if time.Since(*a.HeartbeatTime) < 5 * time.Minute {
a.IsOnline = true
}
if a.IP != "" {
json.Unmarshal([]byte(a.IP), &a.IPList)
}
if a.Disk != "" {
json.Unmarshal([]byte(a.Disk), &a.Disks)
}
}
type AgentManager struct{}
func NewAgentManager() *AgentManager {
return &AgentManager{}
}
func (m *AgentManager) CreateOrReplace(agent *Agent) (*Agent, bool, error) {
now := time.Now()
ormer := orm.NewOrm()
orgAgent := &Agent{UUID: agent.UUID}
if created, _, err := ormer.ReadOrCreate(orgAgent, "UUID"); err != nil {
return nil, false, err
} else {
orgAgent.Hostname = agent.Hostname
orgAgent.IP = agent.IP
orgAgent.OS = agent.OS
orgAgent.CPU = agent.CPU
orgAgent.RAM = agent.RAM
orgAgent.Disk = agent.Disk
orgAgent.BootTime = agent.BootTime
orgAgent.Time = agent.Time
orgAgent.DeletedTime = nil
orgAgent.HeartbeatTime = &now
ormer.Update(orgAgent)
return orgAgent, created, nil
}
}
func (m *AgentManager) Heartbeat(uuid string) {
orm.NewOrm().QueryTable(&Agent{}).Filter("UUID__exact", uuid).Update(orm.Params{"HeartbeatTime": time.Now(), "DeletedTime": nil})
}
func (m *AgentManager) Query(q string, start int64, length int) ([]*Agent, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&Agent{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
qtotal := total
if q != "" {
query := orm.NewCondition()
query = query.Or("hostname__icontains", q)
query = query.Or("ip__icontains", q)
query = query.Or("os__icontains", q)
query = query.Or("arch__icontains", q)
condition = condition.AndCond(query)
qtotal, _ = queryset.SetCond(condition).Count()
}
var result []*Agent
queryset.SetCond(condition).RelatedSel().Limit(length).Offset(start).All(&result)
for _, agent := range result {
agent.Patch()
}
return result, total, qtotal
}
func (m *AgentManager) DeleteById(id int) error {
orm.NewOrm().QueryTable(&Agent{}).Filter("Id__exact", id).Update(orm.Params{"DeletedTime": time.Now()})
return nil
}
var DefaultAgentManager = NewAgentManager()
func init() {
orm.RegisterModel(new(Agent))
}
D:\Workspace\Go\src\gocmdb\server\models\log.go
package models
import (
"encoding/json"
"time"
"github.com/astaxie/beego/orm"
)
const (
LOGResource = 0X0001
)
type Log struct {
UUID string `json:"uuid"`
Type int `json:"type"`
Msg string `json:"msg"`
Time *time.Time `json:"time"`
}
type LogManager struct{}
func NewLogManager() *LogManager {
return &LogManager{}
}
func (m *LogManager) Create(log *Log) {
switch log.Type {
case LOGResource:
resource := &Resource{}
if err := json.Unmarshal([]byte(log.Msg), resource); err == nil {
DefaultResourceManager.Create(log, resource)
}
}
}
type Resource struct {
Id int `orm:"column(id);" json:"id"`
UUID string `orm:"column(uuid);size(64);" json:"uuid"`
Load string `orm:"column(load);size(1024);" json:"load"`
CPUPrecent float64 `orm:"column(cpu_percent);" json:"cpu_percent"`
RAMPrecent float64 `orm:"column(ram_percent);" json:"ram_percent"`
DiskPrecent string `orm:"column(disk_percent);size(4096);" json:"disk_percent"`
Time *time.Time `orm:"column(time);" json:"time"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
}
type ResourceManager struct{}
func NewResourceManager() *ResourceManager {
return &ResourceManager{}
}
func (m *ResourceManager) Create(log *Log, resource *Resource) {
resource.UUID = log.UUID
resource.Time = log.Time
orm.NewOrm().Insert(resource)
}
func (m *ResourceManager) Query(q string, start int64, length int) ([]*Resource, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&Resource{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
qtotal := total
if q != "" {
query := orm.NewCondition()
condition = condition.AndCond(query)
qtotal, _ = queryset.SetCond(condition).Count()
}
var result []*Resource
queryset.SetCond(condition).OrderBy("-created_time").Limit(length).Offset(start).All(&result)
return result, total, qtotal
}
func (m *ResourceManager) Trend(uuid string) []*Resource {
endTime := time.Now()
startTime := endTime.Add(-1 * time.Hour)
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
condition = condition.And("uuid__exact", uuid)
condition = condition.And("created_time__gte", startTime)
var items []*Resource
orm.NewOrm().QueryTable(new(Resource)).SetCond(condition).OrderBy("created_time").All(&items)
var itemMap map[string]*Resource = make(map[string]*Resource)
for _, item := range items {
itemMap[item.CreatedTime.Format("2006-01-02 15:04")] = item
}
var result []*Resource = make([]*Resource, 0, len(items))
for startTime.Before(endTime) {
key := startTime.Format("2006-01-02 15:04")
if item, ok := itemMap[key]; ok {
result = append(result, item)
} else {
result = append(result, &Resource{CreatedTime: &startTime})
}
startTime = startTime.Add(time.Minute)
}
return result
}
var DefaultLogManager = NewLogManager()
var DefaultResourceManager = NewResourceManager()
func init() {
orm.RegisterModel(new(Resource))
}
D:\Workspace\Go\src\gocmdb\server\routers\router.go
v2 "gocmdb/server/controllers/api/v2"
v2Namespace := beego.NewNamespace("/v2",
beego.NSRouter("api/heartbeat/:uuid/", &v2.APIController{}, "*:Heartbeat"),
beego.NSRouter("api/register/:uuid/", &v2.APIController{}, "*:Register"),
beego.NSRouter("api/log/:uuid/", &v2.APIController{}, "*:Log"),
)
beego.AddNamespace(v2Namespace)D:\Workspace\Go\src\gocmdb\server\controllers\api\v2\api.go
package v1
import (
"encoding/json"
"gocmdb/server/controllers/api"
"gocmdb/server/models"
"github.com/beego/beego"
)
type APIController struct {
api.BaseController
}
func (c *APIController) Prepare() {
c.BaseController.Prepare()
if beego.AppConfig.String("agent::token") != c.Ctx.Input.Header("Token") {
c.Data["json"] = map[string]interface{}{
"code": 400,
"text": "Token不正确",
"result": nil,
}
c.ServeJSON()
c.StopRun()
}
}
func (c *APIController) Heartbeat() {
models.DefaultAgentManager.Heartbeat(c.Ctx.Input.Param(":uuid"))
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "成功",
"result": nil,
}
c.ServeJSON()
}
func (c *APIController) Register() {
rt := map[string]interface{}{
"code": 400,
"text": "成功",
"result": nil,
}
agent := &models.Agent{}
if err := json.Unmarshal(c.Ctx.Input.RequestBody, agent); err == nil {
agent, created, err := models.DefaultAgentManager.CreateOrReplace(agent)
if err == nil {
rt = map[string]interface{}{
"code": 200,
"text": "成功",
"result": map[string]interface{}{
"created": created,
"agent": agent,
},
}
} else {
rt["text"] = err.Error()
}
} else {
rt["text"] = err.Error()
}
c.Data["json"] = rt
c.ServeJSON()
}
func (c *APIController) Log() {
log := &models.Log{}
if err := json.Unmarshal(c.Ctx.Input.RequestBody, log); err == nil {
models.DefaultLogManager.Create(log)
}
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "成功",
"result": nil,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\views\agent_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\agent_page\index.script.html
D:\Workspace\Go\src\gocmdb\server\views\resource_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\resource_page\index.script.html
D:\Workspace\Go\src\gocmdb\server\alarm.go
package main
import (
"flag"
"fmt"
"os"
"time"
"github.com/astaxie/beego"
"github.com/astaxie/beego/orm"
_ "github.com/go-sql-driver/mysql"
_ "gocmdb/server/routers"
)
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: alarm -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/alarm.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
go func() {
// 离线告警
for now := range time.Tick(time.Minute) {
fmt.Println("离线告警", now)
offlineTime := 5
endTime := now.Add(-1 * time.Duration(offlineTime) * time.Minute) // 5 根据配置
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid from agent where deleted_time is null and heartbeat_time < ?", endTime).Values(&result)
fmt.Println(result)
}
}()
go func() {
// CPU使用率
for now := range time.Tick(time.Minute) {
fmt.Println("CPU使用率告警", now)
windowTime := 5
cpuThreshold := 30
cpuCounter := 3
startTime := now.Add(-1 * time.Duration(windowTime) * time.Minute) // 5 根据配置
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid, count(*) as cnt from resource where deleted_time is null and created_time >= ? and cpu_percent >= ? group by uuid having count(*) >= ?", startTime, cpuThreshold, cpuCounter).Values(&result)
fmt.Println(result)
}
}()
// 内存使用率
for now := range time.Tick(time.Minute) {
fmt.Println("内存使用率告警", now)
windowTime := 5
ramThreshold := 40
ramCounter := 3
startTime := now.Add(-1 * time.Duration(windowTime) * time.Minute) // 5 根据配置
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid, count(*) as cnt from resource where deleted_time is null and created_time >= ? and ram_percent >= ? group by uuid having count(*) >= ?", startTime, ramThreshold, ramCounter).Values(&result)
fmt.Println(result)
}
}
server-side development
D:\Workspace\Go\src\gocmdb\server\conf\app.conf
include "smtp.conf"
include "sms.conf"
D:\Workspace\Go\src\gocmdb\server\conf\sms.conf
[sms]
endpoint=sms.tencentcloudapi.com
secretId=AKIDA8ta3JL6pKaicHsxxtkOmvpVv59y3u0r
secretKey=pSr30c0JOEo9F6h8UeaGkFXeXQ7bsuiH
appid=1400287583
sign=imuk网
templateOfflineId=510285
templateRamId=510288
templateCPUId=510287
phones=8613129091210;8613610847443
D:\Workspace\Go\src\gocmdb\server\conf\smtp.conf
[smtp]
host=smtp.qq.com
port=465
[email protected]
password=bsmujneemttzbbfe
[email protected]
Alarm reminder notification
Display the latest 10 unhandled alarm information
Dashboard
Number of online terminals/number of offline terminals/total number of unprocessed alarms
Distribution of Unprocessed Alarms
Trends of various status alarms in the last 7 days
Task
Example: Obtain information about the current execution process of the terminal
task creation
task acquisition
task execution
Task result upload
Storage of task results
Redis
Chinese documentation: http://doc.redisfans.com/
Common Data Types & Operations
string
the list
hash
gather
ordered set
Go operation Redis
Third-party library: github.com/gomodule/redigo/redis
1. Mail sending server stmp protocol
socket server
ip, port
authentication (username/password)
Send EmailForm
To
CC
title
content
html/text
Provide web service to send mail
Call web service
match
url
Authentication (Token)
to, subject, content
send email
limit frequency
Count 1 hour in memory
uuid type count createTime 1 2
uuid+type
Judging createTime < now - 1h
and recounting
, otherwise
counting + 1
Use third-party memory storage
redis/mem
uuid type ttl
1. Mobile phone ==> SMS modem
2. SMS service API
Alibaba Cloud
Tencent Cloud
a := 1
b := &a
c := []*int{}
c = append(b)
c = append(b)
c = append(b)
*b = 3
a
*c[0], *c[1], *c[2]
Problem:
SMS sending failed, no retry mechanism
Make an SMS service (web)
to restrict others
from retrying in order to ensure the success of SMS sending
code valid once
Login with email verification code
Use mobile phone verification code to log in
1. User fills in mobile phone number (send verification code) => submit request to server (phone)
2. Verify that the mobile phone number
is not valid => mobile phone number is wrong
=> can send text messages
to limit the frequency ( Sent once every 1 minute)
Yes sent => Prompt sent to
No sent
=> Generate a random verification code
=> Send to the user and store the login verification code corresponding to the mobile phone Create/update phone code code_created_time (code expiration time is 5 minutes)
3. User Fill in the verification code to log in => submit data to the server (phone, code)
phone search data
no data => mobile phone number or verification code is wrong
=> verification code is ""
is empty => used, mobile phone number or verification Code error
is not empty => verify whether the code is correct
or not => mobile phone number or verification code is incorrect
=> determine whether code_created_time has expired
Expired => mobile phone number or verification code error
is not expired => successful login
to clear the code
Delete Record
Audit Log/Operation Log
Alarm management
time, terminal, type, status
Alarm processing Open the dialog to modify the status and reason
The number of each alarm and each processing status per day in the last 7 days
where deleted_time alarmed_time
group days, alarm type, status
D:\Workspace\Go\src\gocmdb\server\alarm.go
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: alarm -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/alarm.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
host := beego.AppConfig.String("smtp::host")
port, _ := beego.AppConfig.Int("smtp::port")
user := beego.AppConfig.String("smtp::user")
password := beego.AppConfig.String("smtp::password")
to := beego.AppConfig.Strings("smtp::to")
emailSender := utils.NewEmail(host, port, user, password)
smsSender := utils.NewSms(
beego.AppConfig.String("sms::endpoint"),
beego.AppConfig.String("sms::secretId"),
beego.AppConfig.String("sms::secretKey"),
beego.AppConfig.String("sms::appid"),
beego.AppConfig.String("sms::sign"),
)
templateOfflineId := beego.AppConfig.String("sms::templateOfflineId")
templateCPUId := beego.AppConfig.String("sms::templateCPUId")
templateRamId := beego.AppConfig.String("sms::templateRamId")
phones := beego.AppConfig.Strings("sms::phones")
go func() {
// 离线告警
offlineTime := 5
noticeWindowTime := 60
noticeCounter := int64(2)
for now := range time.Tick(time.Minute) {
beego.Debug("离线告警", now)
endTime := now.Add(-1 * time.Duration(offlineTime) * time.Minute) // 5 根据配置
noticeStartTime := now.Add(-1 * time.Duration(noticeWindowTime) * time.Minute)
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid,heartbeat_time from agent where deleted_time is null and heartbeat_time < ?", endTime).Values(&result)
for _, line := range result {
uuid, _ := line["uuid"].(string)
heartbeat_time, _ := line["heartbeat_time"].(string)
content := fmt.Sprintf("终端[%s]最后一次发送心跳时间为%s, 已超过离线时间%d分钟", uuid, heartbeat_time, offlineTime)
alarmCnt := models.DefaultAlarmManager.GetCountByUuidAndType(uuid, models.AlarmTypeOffline, noticeStartTime)
if alarmCnt >= noticeCounter {
beego.Info(fmt.Sprintf("通知次数(%d)超过限制(%d), %s", alarmCnt, noticeCounter, content))
continue
}
emailErr := emailSender.Send(to, "[CMDB]终端离线告警", content, []string{})
params := []string{uuid, heartbeat_time, strconv.Itoa(offlineTime)}
smsErr := smsSender.Send(templateOfflineId, phones, params)
beego.Info("终端离线告警: ", content, ", email通知:", emailErr, ", sms通知:", smsErr)
models.DefaultAlarmManager.Create(uuid, models.AlarmTypeOffline, content, now)
}
}
}()
go func() {
windowTime := 5
cpuThreshold := 10
cpuCounter := 3
noticeWindowTime := 60
noticeCounter := int64(2)
// CPU使用率
for now := range time.Tick(time.Minute) {
beego.Debug("CPU使用率告警", now)
startTime := now.Add(-1 * time.Duration(windowTime) * time.Minute) // 5 根据配置
noticeStartTime := now.Add(-1 * time.Duration(noticeWindowTime) * time.Minute)
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid, count(*) as cnt from resource where deleted_time is null and created_time >= ? and cpu_percent >= ? group by uuid having count(*) >= ?", startTime, cpuThreshold, cpuCounter).Values(&result)
for _, line := range result {
uuid, _ := line["uuid"].(string)
cntString, _ := line["cnt"].(string)
cnt, _ := strconv.Atoi(cntString)
content := fmt.Sprintf("终端[%s]在最近%d分钟内CPU使用率大于%d%%的次数为%d, 已超过%d次", uuid, windowTime, cpuThreshold, cnt, cpuCounter)
alarmCnt := models.DefaultAlarmManager.GetCountByUuidAndType(uuid, models.AlarmTypeCPU, noticeStartTime)
if alarmCnt >= noticeCounter {
beego.Info(fmt.Sprintf("通知次数(%d)超过限制(%d), %s", alarmCnt, noticeCounter, content))
continue
}
emailErr := emailSender.Send(to, "[CMDB]终端CPU告警", content, []string{})
params := []string{uuid, strconv.Itoa(windowTime), strconv.Itoa(cpuThreshold), strconv.Itoa(cnt), strconv.Itoa(cpuCounter)}
smsErr := smsSender.Send(templateCPUId, phones, params)
beego.Info("终端CPU告警: ", content, ", email通知:", emailErr, ", sms通知:", smsErr)
models.DefaultAlarmManager.Create(uuid, models.AlarmTypeCPU, content, now)
}
}
}()
// 内存使用率
windowTime := 5
ramThreshold := 10
ramCounter := 3
noticeWindowTime := 60
noticeCounter := int64(2)
for now := range time.Tick(time.Minute) {
beego.Debug("内存使用率告警", now)
startTime := now.Add(-1 * time.Duration(windowTime) * time.Minute) // 5 根据配置
noticeStartTime := now.Add(-1 * time.Duration(noticeWindowTime) * time.Minute) // 5 根据配置
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid, count(*) as cnt from resource where deleted_time is null and created_time >= ? and ram_percent >= ? group by uuid having count(*) >= ?", startTime, ramThreshold, ramCounter).Values(&result)
for _, line := range result {
uuid, _ := line["uuid"].(string)
cntString, _ := line["cnt"].(string)
cnt, _ := strconv.Atoi(cntString)
content := fmt.Sprintf("终端[%s]在最近%d分钟内内存使用率大于%d%%的次数为%d, 已超过%d次", uuid, windowTime, ramThreshold, cnt, ramCounter)
alarmCnt := models.DefaultAlarmManager.GetCountByUuidAndType(uuid, models.AlarmTypeRam, noticeStartTime)
if alarmCnt >= noticeCounter {
beego.Info(fmt.Sprintf("通知次数(%d)超过限制(%d), %s", alarmCnt, noticeCounter, content))
continue
}
emailErr := emailSender.Send(to, "[CMDB]终端内存告警", content, []string{})
params := []string{uuid, strconv.Itoa(windowTime), strconv.Itoa(ramThreshold), strconv.Itoa(cnt), strconv.Itoa(ramCounter)}
smsErr := smsSender.Send(templateRamId, phones, params)
beego.Info("终端内存告警: ", content, ", email通知:", emailErr, ", sms通知:", smsErr)
models.DefaultAlarmManager.Create(uuid, models.AlarmTypeRam, content, now)
}
}
}
D:\Workspace\Go\src\gocmdb\server\models\alarm.go
package models
import (
"fmt"
"strconv"
"time"
"github.com/astaxie/beego/orm"
)
type Alarm struct {
Id int `orm:"column(id);" json:"id"`
UUID string `orm:"column(uuid);size(64);" json:"uuid"`
Type int `orm:"column(type)" json:"type"`
Content string `orm:"column(content);type(text);" json:"content"`
AlarmedTime *time.Time `orm:"column(alarmed_time);" json:"alarmed_time"`
Status int `orm:"column(status);" json:"status"`
Reason string `orm:"column(reason);type(text);" json:"reason"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
}
type AlarmManager struct{}
func NewAlarmManager() *AlarmManager {
return &AlarmManager{}
}
func (m *AlarmManager) Create(uuid string, typ int, content string, alarmedTime time.Time) error {
alarm := &Alarm{
UUID: uuid,
Type: typ,
Content: content,
AlarmedTime: &alarmedTime,
Status: AlarmStatusNew,
Reason: "",
}
_, err := orm.NewOrm().Insert(alarm)
return err
}
func (m *AlarmManager) GetCountByUuidAndType(uuid string, typ int, startTime time.Time) int64 {
cnt, _ := orm.NewOrm().QueryTable(new(Alarm)).Filter("uuid__exact", uuid).Filter("type__exact", typ).Filter("alarmed_time__gte", startTime).Filter("deleted_time__isnull", true).Count()
return cnt
}
func (m *AlarmManager) GetNotification(limit int) (int64, []*Alarm) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(new(Alarm))
cnt, _ := queryset.Filter("status__exact", AlarmStatusNew).Filter("deleted_time__isnull", true).Count()
var result []*Alarm
queryset.Filter("status__exact", AlarmStatusNew).Filter("deleted_time__isnull", true).OrderBy("-alarmed_time").Limit(limit).All(&result)
return cnt, result
}
func (m *AlarmManager) GetCountForNoComplete() int64 {
// total, _ := orm.NewOrm().QueryTable(new(Alarm)).Filter("deleted_time__isnull", true).Exclude("status__exact", AlarmStatusComplete).Count()
// fmt.Println(total)
total, _ := orm.NewOrm().QueryTable(new(Alarm)).Filter("deleted_time__isnull", true).Filter("status__in", AlarmStatusNew, AlarmStatusDoing).Count()
return total
}
func (m *AlarmManager) GetStatForNotComplete() map[string]int64 {
var lines []orm.Params
orm.NewOrm().Raw("select type, count(*) as cnt from alarm where deleted_time is null and status in (?, ?) group by type", []int{AlarmStatusDoing, AlarmStatusNew}).Values(&lines)
result := map[string]int64{}
for _, line := range lines {
typ := line["type"].(string)
cntString := line["cnt"].(string)
cnt, _ := strconv.ParseInt(cntString, 10, 64)
result[typ] = cnt
}
return result
}
func (m *AlarmManager) GetLastestNStat(day int) ([]string, map[string][]int64) {
endTime := time.Now()
startTime := endTime.Add(-24*time.Duration(day-1)*time.Hour - 1)
var lines []orm.Params
orm.NewOrm().Raw("select date_format(alarmed_time, '%Y-%m-%d') as day, type, status, count(*) as cnt from alarm where deleted_time is null and alarmed_time >= ? group by day, type, status", startTime).Values(&lines)
//key type+status = day : cnt
tempStat := map[string]map[string]int64{}
for _, line := range lines {
day, _ := line["day"].(string)
status, _ := line["status"].(string)
typ, _ := line["type"].(string)
cntString, _ := line["cnt"].(string)
cnt, _ := strconv.ParseInt(cntString, 10, 64)
key := fmt.Sprintf("%s-%s", typ, status)
if _, ok := tempStat[key]; !ok {
tempStat[key] = map[string]int64{}
}
tempStat[key][day] = cnt
}
// type+status : [1, 2, 3, 3, 4]
// "int-int"
days := []string{}
result := map[string][]int64{}
for startTime.Before(endTime) {
day := startTime.Format("2006-01-02")
days = append(days, day)
for _, typ := range []int{AlarmTypeOffline, AlarmTypeCPU, AlarmTypeRam} {
for _, status := range []int{AlarmStatusNew, AlarmStatusDoing, AlarmStatusComplete} {
key := fmt.Sprintf("%d-%d", typ, status)
value := int64(0)
if stat, ok := tempStat[key]; ok {
value = stat[day]
} else {
}
result[key] = append(result[key], value)
}
}
startTime = startTime.Add(24 * time.Hour)
}
return days, result
}
var DefaultAlarmManager = NewAlarmManager()
func init() {
orm.RegisterModel(new(Alarm))
}
D:\Workspace\Go\src\gocmdb\server\models\enum.go
package models
const (
StatusUnlock = 0
StatusLock = 1
)
const (
AlarmTypeOffline = iota
AlarmTypeCPU
AlarmTypeRam
)
const (
AlarmStatusNew = iota
AlarmStatusDoing
AlarmStatusComplete
)
D:\Workspace\Go\src\gocmdb\server\utils\email.go
package utils
import (
"gopkg.in/gomail.v2"
)
type Email struct {
host string
port int
user string
password string
}
func NewEmail(host string, port int, user string, password string) *Email {
return &Email{
host: host,
port: port,
user: user,
password: password,
}
}
func (e *Email) Send(to []string, subject string, msg string, attaches []string) error {
m := gomail.NewMessage()
m.SetHeader("From", e.user)
m.SetHeader("To", to...)
m.SetHeader("Subject", subject)
m.SetBody("text/html", msg)
for _, attach := range attaches {
m.Attach(attach)
}
d := gomail.NewDialer(e.host, e.port, e.user, e.password)
return d.DialAndSend(m)
}
D:\Workspace\Go\src\gocmdb\server\utils\sms.go
package utils
import (
"fmt"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
sms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms/v20190711"
)
type Sms struct {
endpoint string
secretId string
secretKey string
appid string
sign string
}
func NewSms(endpoint, secretId, secretKey, appid, sign string) *Sms {
return &Sms{
endpoint: endpoint,
secretId: secretId,
secretKey: secretKey,
appid: appid,
sign: sign,
}
}
func (s *Sms) Send(templateId string, phones []string, params []string) error {
credential := common.NewCredential(s.secretId, s.secretKey)
cpf := profile.NewClientProfile()
cpf.HttpProfile.Endpoint = s.endpoint
client, err := sms.NewClient(credential, "", cpf)
if err != nil {
return err
}
phoneSet := []*string{}
for _, phone := range phones {
temp := phone
phoneSet = append(phoneSet, &temp)
}
paramSet := []*string{}
for _, param := range params {
temp := string([]rune(param)[:12])
paramSet = append(paramSet, &temp)
}
request := sms.NewSendSmsRequest()
request.PhoneNumberSet = phoneSet
request.TemplateID = &templateId
request.SmsSdkAppid = &s.appid
request.Sign = &s.sign
request.TemplateParamSet = paramSet
response, err := client.SendSms(request)
if err != nil {
return err
}
fmt.Printf("%s", response.ToJsonString())
return err
}D:\Workspace\Go\src\gocmdb\server\controllers\dashboard.go
package controllers
import (
"gocmdb/server/controllers/auth"
"gocmdb/server/models"
)
type DashboardPageController struct {
LayoutController
}
func (c *DashboardPageController) Index() {
c.Data["menu"] = "dashboard"
c.TplName = "dashboard_page/index.html"
c.LayoutSections["LayoutScript"] = "dashboard_page/index.script.html"
}
type DashboardController struct {
auth.LoginRequiredController
}
func (c *DashboardController) Stat() {
onlineCnt, offlineCnt := models.DefaultAgentManager.GetStat()
alarm_trend_days, alarm_trend_data := models.DefaultAlarmManager.GetLastestNStat(7)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": map[string]interface{}{
"agent_offline_count": offlineCnt,
"agent_online_count": onlineCnt,
"alarm_count": models.DefaultAlarmManager.GetCountForNoComplete(),
"alarm_dist": models.DefaultAlarmManager.GetStatForNotComplete(),
"alarm_trend": map[string]interface{}{
"days": alarm_trend_days,
"data": alarm_trend_data,
},
},
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\models\agent.go
func (m *AgentManager) GetStat() (int64, int64) {
now := time.Now()
onlineTime := now.Add(-5 * time.Minute)
queryset := orm.NewOrm().QueryTable(new(Agent)).Filter("deleted_time__isnull", true)
onlineCnt, _ := queryset.Filter("heartbeat_time__gte", onlineTime).Count()
offlineCnt, _ := queryset.Filter("heartbeat_time__lt", onlineTime).Count()
return onlineCnt, offlineCnt
}D:\Workspace\Go\src\gocmdb\server\routers\router.go
// 认证
beego.Router("/", &controllers.IndexController{}, "get:Index")
// 认证
beego.AutoRouter(&auth.AuthController{})
// Dashboard
beego.AutoRouter(&controllers.DashboardPageController{})
// Dashboard
beego.AutoRouter(&controllers.DashboardController{})
// 用户页面
beego.AutoRouter(&controllers.UserPageController{})D:\Workspace\Go\src\gocmdb\server\controllers\default.go
package controllers
import (
"github.com/astaxie/beego"
"net/http"
)
type IndexController struct {
beego.Controller
}
func (c *IndexController) Index() {
c.Redirect(beego.URLFor(beego.AppConfig.String("home")), http.StatusFound)
}
D:\Workspace\Go\src\gocmdb\server\views\dashboard_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\dashboard_page\index.script.html
Task
Terminal
Plugin
Plugin Parameters
Status
Start Time
End Time
Task Result
Task
Result
Failure Reason
AGENT HOW TO GET TASKS
Request tasks from the server regularly and ask the server every 10s, what tasks (newly created) do I have to execute
(ENS) I will check, 2 tasks, mark the task has been returned to the AGENT (executing), return to the AGENT
task executes
each task => plugin
plugin management => responsible for execution
ENS => (Pipeline) => Manager communication
Redis common operations
auth authentication
ping test
Processing
type, del, keys, ttl, expire, exists for KEY
string for each data type
set
get
mset
mget
setnx
incr
incrby
decr
decrby
list
lpush
rpush
lpop
rpop
lrange
llen
brpop
blpop
hash
hset
hget
hmset
hmget
hsetnx
hdel
hexists
hlen
hgetall
set
sadd
scard
smembers
sismember
srem
sorted
setzadd
zcard
zrange
zrevrange
zrangebyscore
zrevrangebyscore
zrem
Publish
subscribe subscribe
publish
operation and maintenance
heartbeat => db
heartbeat => zset
uuid score time unix
zrangebyscore uuid
now - 5 now
supervisor
1. yum install/pip install
2. systemd
deployment architecture
cache
architecture diagram
process communication
configuration
Task
Task delivery & execution process
Redis
NGINX
Cross Compilation & Conditional Compilation
other
Websocket
elasticsearch
Go operation Redis
Third-party library: github.com/gomodule/redigo/redis
string manipulation
KEY operation
LIST operation
HASH operation
SET operation
ZSET operation
Publish Subscribe Operation
REDIS application
Cache: such as SESSION
Process communication: queue (producer, consumer)
choose the master
Ranking: Ordered Set
Message Broadcasting: Publish Subscribe
BEEGO uses REDIS to store SESSION
configuration
import package
Use Redis cache in BEEGO
Use Redis cache in BEEGO
Beego web deployment
go build web.go
Copy web.exe static views conf to the deployment environment
start web program
NGINX
Nginx is a high performance HTTP and reverse proxy web server
load balancing algorithm
polling
assign weight
ip_hash
fair
url_hash
NGINX proxy Beego Web application
Beego web can be modified only for local access (httpaddr)
Configure nginx.conf and start
daemon off;
#user nobody;
worker_processes 4;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
pid logs/nginx.pid;
events {
worker_connections 10240;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream app_server_cmdb {
server 127.0.0.1:8888;
}
server {
listen 80;
server_name cmdb;
charset utf-8;
error_log logs/cmdb.error.log debug;
access_log logs/cmdb.access.log main;
root D:\\codes\\gocmdb-deploy;
gzip on;
gzip_min_length 1024;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
gzip_vary on;
client_body_temp_path temp/client_body_temp_cmdb 1 2 ;
proxy_temp_path temp/proxy_temp_cmdb 1 2;
fastcgi_temp_path temp/fastcgi_temp_cmdb 1 2;
uwsgi_temp_path temp/uwsgi_temp_cmdb 1 2;
scgi_temp_path temp/scgi_temp_cmdb 1 2;
location /static/ {
alias D:\\codes\\gocmdb-deploy\\static\\;
expires 1d;
access_log off;
}
location / {
try_files $uri @proxy_to_app;
}
location @proxy_to_app {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_pass http://app_server_cmdb;
}
}
upstream app_server_test {
server 127.0.0.1:9990 weight=2;
server 127.0.0.1:9991;
}
server {
listen 8080;
server_name test;
charset utf-8;
error_log logs/test.error.log debug;
access_log logs/test.access.log main;
root D:\\codes\\test;
gzip on;
gzip_min_length 1024;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
gzip_vary on;
client_body_temp_path temp/client_body_temp_cmdb 1 2 ;
proxy_temp_path temp/proxy_temp_cmdb 1 2;
fastcgi_temp_path temp/fastcgi_temp_cmdb 1 2;
uwsgi_temp_path temp/uwsgi_temp_cmdb 1 2;
scgi_temp_path temp/scgi_temp_cmdb 1 2;
location /static/ {
alias D:\\codes\\test\\static\\;
expires 1d;
access_log off;
}
location / {
try_files $uri @proxy_to_app;
}
location @proxy_to_app {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_pass http://app_server_test;
}
}
}
NGINX load balancing usage
Start the cache program for ports 9990 and 9991 respectively
Cross Compilation & Conditional Compilation
Cross compilation is only available when not using CGO
$GOOS.go
$GOOS_$GOARCH.go
Performance Testing
API to DB
API to Redis
agent development
D:\Workspace\Go\src\gocmdb\agent\config\config.go
type Config struct {
Task chan interface{}
TaskResult chan interface{}
}
return &Config{
Task: make(chan interface{}, 128),
TaskResult: make(chan interface{}, 128),
}, nilD:\Workspace\Go\src\gocmdb\agent\ens\ens.go
func (s *ENS) Start() {
logrus.Info("ENS 开始运行")
headers := req.Header{"Token": s.conf.Token}
request := req.New()
go func() {
endpoint := fmt.Sprintf("%s/result/%s/", s.conf.Endpoint, s.conf.UUID)
for evt := range s.conf.TaskResult {
response, err := request.Post(endpoint, req.BodyJSON(evt), headers)
if err == nil {
result := map[string]interface{}{}
response.ToJSON(&result)
logrus.WithFields(logrus.Fields{
"taskResult": evt,
"result": result,
}).Debug("任务结果上传成功")
} else {
logrus.WithFields(logrus.Fields{
"taskResult": evt,
"error": err,
}).Error("任务结果上传失败")
}
}
}()
go func() {
endpoint := fmt.Sprintf("%s/task/%s/", s.conf.Endpoint, s.conf.UUID)
for now := range time.Tick(10 * time.Second) {
response, err := request.Get(endpoint, req.QueryParam{"time": now.Unix()}, headers)
if err == nil {
result := map[string]interface{}{}
response.ToJSON(&result)
logrus.WithFields(logrus.Fields{
"result": result,
}).Debug("获取任务成功")
tasks, _ := result["result"].([]interface{})
for _, taskMap := range tasks {
if task, err := entity.NewTask(taskMap); err == nil {
s.conf.Task <- task
}
}
} else {
logrus.WithFields(logrus.Fields{
"error": err,
}).Error("获取任务失败")
}
}
}()
}D:\Workspace\Go\src\gocmdb\agent\entity\task.go
package entity
import (
"encoding/json"
)
type Task struct {
Id int `json:"id"`
Plugin string `json:"plugin"`
Params string `json:"params"`
Timeout int `json:"timeout"`
}
func NewTask(taskMap interface{}) (Task, error) {
var task Task
if taskBytes, err := json.Marshal(taskMap); err != nil {
return task, err
} else if err = json.Unmarshal(taskBytes, &task); err != nil {
return task, err
}
return task, nil
}
type Result struct {
TaskId int `json:"task_id"`
Status int `json:"status"`
Result string `json:"result"`
Err string `json:"err"`
}
func NewResult(task Task, result interface{}, err error) Result {
bytes, _ := json.Marshal(result)
errInfo := ""
status := 0
if err != nil {
status = 1
errInfo = err.Error()
}
return Result{
TaskId: task.Id,
Status: status,
Result: string(bytes),
Err: errInfo,
}
}
D:\Workspace\Go\src\gocmdb\agent\plugins\init\task.go
package init
import (
"gocmdb/agent/plugins"
"gocmdb/agent/plugins/task"
)
func init() {
plugins.DefaultManager.RegisterTask(&task.Process{})
}
D:\Workspace\Go\src\gocmdb\agent\plugins\task\process.go
package task
import (
"gocmdb/agent/config"
"github.com/shirou/gopsutil/process"
)
type Process struct {
conf *config.Config
}
func (p *Process) Name() string {
return "process"
}
func (p *Process) Init(conf *config.Config) {
p.conf = conf
}
func (p *Process) Call(params string) (interface{}, error) {
pids, err := process.Pids()
if err != nil {
return nil, err
}
rs := make([]map[string]interface{}, len(pids))
for index, pid := range pids {
ps, err := process.NewProcess(pid)
if err != nil {
continue
}
name, _ := ps.Name()
exe, _ := ps.Exe()
cmd, _ := ps.Cmdline()
createdTime, _ := ps.CreateTime()
ppid, _ := ps.Ppid()
cwd, _ := ps.Cwd()
numFDs, _ := ps.NumFDs()
numThreads, _ := ps.NumThreads()
memoryInfo, _ := ps.MemoryInfo()
connections, _ := ps.Connections()
rs[index] = map[string]interface{}{
"pid": pid,
"ppid": ppid,
"name": name,
"exe": exe,
"cmd": cmd,
"createdTime": createdTime,
"cwd": cwd,
"numFDs": numFDs,
"numThreads": numThreads,
"memoryInfo": memoryInfo,
"connections": connections,
}
}
return rs, nil
}
D:\Workspace\Go\src\gocmdb\agent\plugins\base.go
type TaskPlugin interface {
Name() string
Init(*config.Config)
Call(params string) (interface{}, error)
}D:\Workspace\Go\src\gocmdb\agent\plugins\manager.go
package plugins
import (
"errors"
"gocmdb/agent/config"
"time"
"gocmdb/agent/entity"
"github.com/sirupsen/logrus"
)
type Manager struct {
Conf *config.Config
Cycles map[string]CyclePlugin
Tasks map[string]TaskPlugin
}
func NewManager() *Manager {
return &Manager{
Cycles: make(map[string]CyclePlugin),
Tasks: make(map[string]TaskPlugin),
}
}
func (m *Manager) RegisterCycle(p CyclePlugin) {
m.Cycles[p.Name()] = p
logrus.WithFields(logrus.Fields{
"Name": p.Name(),
"Type": "周期型",
}).Info("插件注册")
}
func (m *Manager) RegisterTask(p TaskPlugin) {
m.Tasks[p.Name()] = p
logrus.WithFields(logrus.Fields{
"Name": p.Name(),
"Type": "任务型",
}).Info("插件注册")
}
func (m *Manager) Init(conf *config.Config) {
for name, plugin := range m.Cycles {
plugin.Init(conf)
logrus.WithFields(logrus.Fields{
"Name": name,
}).Info("初始化插件")
}
for name, plugin := range m.Tasks {
plugin.Init(conf)
logrus.WithFields(logrus.Fields{
"Name": name,
}).Info("初始化插件")
}
}
func (m *Manager) Start() {
go m.StartCycle()
go m.StartTask()
}
func (m *Manager) StartCycle() {
for now := range time.Tick(time.Second) {
for name, plugin := range m.Cycles {
if now.After(plugin.NextTime()) {
if evt, err := plugin.Call(); err == nil {
logrus.WithFields(logrus.Fields{
"Name": name,
"Result": evt,
}).Debug("插件执行")
plugin.Pipline() <- evt
} else {
logrus.WithFields(logrus.Fields{
"Name": name,
"error": err,
}).Debug("插件执行失败")
}
}
}
}
}
func (m *Manager) StartTask() {
for task := range m.Conf.Task {
taskObj, _ := task.(entity.Task)
if plugin, ok := m.Tasks[taskObj.Plugin]; !ok {
logrus.WithFields(logrus.Fields{
"task": taskObj,
}).Error("插件执行失败, 插件不存在")
m.Conf.TaskResult <- entity.NewResult(taskObj, nil, errors.New("插件不存在"))
} else {
go func(pluginName string, plugin TaskPlugin) {
result, err := plugin.Call(taskObj.Params)
logrus.WithFields(logrus.Fields{
"Name": pluginName,
"task": taskObj,
"Result": result,
"Err": err,
}).Error("插件执行完成")
m.Conf.TaskResult <- entity.NewResult(taskObj, result, err)
}(plugin.Name(), plugin)
}
}
}
var DefaultManager = NewManager()
server-side development
D:\Workspace\Go\src\gocmdb\server\web.go
package main
import (
"flag"
"fmt"
"os"
"gocmdb/server/models"
_ "gocmdb/server/routers"
"gocmdb/server/utils"
"github.com/astaxie/beego"
"github.com/astaxie/beego/orm"
_ "github.com/go-sql-driver/mysql"
_ "gocmdb/server/cloud/plugins"
_ "github.com/astaxie/beego/session/redis"
)
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
init := flag.Bool("init", false, "init server")
syncdb := flag.Bool("syncdb", false, "sync db")
force := flag.Bool("force", false, "force sync db(drop table)")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: web -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/web.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
// 根据参数选择执行流程
switch {
case *init:
orm.RunSyncdb("default", *force, *verbose)
ormer := orm.NewOrm()
admin := &models.User{Name: "admin", IsSuperman: true}
if err := ormer.Read(admin, "Name"); err == orm.ErrNoRows {
password := utils.RandString(6)
admin.SetPassword(password)
if _, err := ormer.Insert(admin); err == nil {
beego.Informational("初始化admin成功, 默认密码:", password)
} else {
beego.Error("初始化用户失败, 错误:", err)
}
} else {
beego.Informational("admin用户已存在, 跳过")
}
case *syncdb:
orm.RunSyncdb("default", *force, *verbose)
beego.Informational("同步数据库")
default:
beego.Run()
}
}
D:\Workspace\Go\src\gocmdb\server\alarm.go
package main
import (
"flag"
"fmt"
"os"
"os/signal"
"strconv"
"syscall"
"time"
"github.com/astaxie/beego"
"github.com/astaxie/beego/orm"
_ "github.com/go-sql-driver/mysql"
"gocmdb/server/models"
_ "gocmdb/server/routers"
"gocmdb/server/utils"
)
func main() {
// 初始化命令行参数
h := flag.Bool("h", false, "help")
help := flag.Bool("help", false, "help")
verbose := flag.Bool("v", false, "verbose")
flag.Usage = func() {
fmt.Println("usage: alarm -h")
flag.PrintDefaults()
}
// 解析命令行参数
flag.Parse()
if *h || *help {
flag.Usage()
os.Exit(0)
}
// 设置日志到文件
beego.SetLogger("file", `{
"filename" : "logs/alarm.log",
"level" : 7}`,
)
if !*verbose {
//删除控制台日志
beego.BeeLogger.DelLogger("console")
} else {
orm.Debug = true
}
// 初始化orm
orm.RegisterDriver("mysql", orm.DRMySQL)
orm.RegisterDataBase("default", "mysql", beego.AppConfig.String("dsn"))
// 测试数据库连接是否正常
if db, err := orm.GetDB(); err != nil || db.Ping() != nil {
beego.Error("数据库连接错误")
os.Exit(-1)
}
host := beego.AppConfig.String("smtp::host")
port, _ := beego.AppConfig.Int("smtp::port")
user := beego.AppConfig.String("smtp::user")
password := beego.AppConfig.String("smtp::password")
to := beego.AppConfig.Strings("smtp::to")
emailSender := utils.NewEmail(host, port, user, password)
smsSender := utils.NewSms(
beego.AppConfig.String("sms::endpoint"),
beego.AppConfig.String("sms::secretId"),
beego.AppConfig.String("sms::secretKey"),
beego.AppConfig.String("sms::appid"),
beego.AppConfig.String("sms::sign"),
)
templateOfflineId := beego.AppConfig.String("sms::templateOfflineId")
templateCPUId := beego.AppConfig.String("sms::templateCPUId")
templateRamId := beego.AppConfig.String("sms::templateRamId")
phones := beego.AppConfig.Strings("sms::phones")
go func() {
// 离线告警
offlineTime := 5
noticeWindowTime := 60
noticeCounter := int64(2)
for now := range time.Tick(time.Minute) {
beego.Debug("离线告警", now)
endTime := now.Add(-1 * time.Duration(offlineTime) * time.Minute) // 5 根据配置
noticeStartTime := now.Add(-1 * time.Duration(noticeWindowTime) * time.Minute)
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid,heartbeat_time from agent where deleted_time is null and heartbeat_time < ?", endTime).Values(&result)
for _, line := range result {
uuid, _ := line["uuid"].(string)
heartbeat_time, _ := line["heartbeat_time"].(string)
content := fmt.Sprintf("终端[%s]最后一次发送心跳时间为%s, 已超过离线时间%d分钟", uuid, heartbeat_time, offlineTime)
alarmCnt := models.DefaultAlarmManager.GetCountByUuidAndType(uuid, models.AlarmTypeOffline, noticeStartTime)
if alarmCnt >= noticeCounter {
beego.Info(fmt.Sprintf("通知次数(%d)超过限制(%d), %s", alarmCnt, noticeCounter, content))
continue
}
emailErr := emailSender.Send(to, "[CMDB]终端离线告警", content, []string{})
params := []string{uuid, heartbeat_time, strconv.Itoa(offlineTime)}
smsErr := smsSender.Send(templateOfflineId, phones, params)
beego.Info("终端离线告警: ", content, ", email通知:", emailErr, ", sms通知:", smsErr)
models.DefaultAlarmManager.Create(uuid, models.AlarmTypeOffline, content, now)
}
}
}()
go func() {
windowTime := 5
cpuThreshold := 10
cpuCounter := 3
noticeWindowTime := 60
noticeCounter := int64(2)
// CPU使用率
for now := range time.Tick(time.Minute) {
beego.Debug("CPU使用率告警", now)
startTime := now.Add(-1 * time.Duration(windowTime) * time.Minute) // 5 根据配置
noticeStartTime := now.Add(-1 * time.Duration(noticeWindowTime) * time.Minute)
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid, count(*) as cnt from resource where deleted_time is null and created_time >= ? and cpu_percent >= ? group by uuid having count(*) >= ?", startTime, cpuThreshold, cpuCounter).Values(&result)
for _, line := range result {
uuid, _ := line["uuid"].(string)
cntString, _ := line["cnt"].(string)
cnt, _ := strconv.Atoi(cntString)
content := fmt.Sprintf("终端[%s]在最近%d分钟内CPU使用率大于%d%%的次数为%d, 已超过%d次", uuid, windowTime, cpuThreshold, cnt, cpuCounter)
alarmCnt := models.DefaultAlarmManager.GetCountByUuidAndType(uuid, models.AlarmTypeCPU, noticeStartTime)
if alarmCnt >= noticeCounter {
beego.Info(fmt.Sprintf("通知次数(%d)超过限制(%d), %s", alarmCnt, noticeCounter, content))
continue
}
emailErr := emailSender.Send(to, "[CMDB]终端CPU告警", content, []string{})
params := []string{uuid, strconv.Itoa(windowTime), strconv.Itoa(cpuThreshold), strconv.Itoa(cnt), strconv.Itoa(cpuCounter)}
smsErr := smsSender.Send(templateCPUId, phones, params)
beego.Info("终端CPU告警: ", content, ", email通知:", emailErr, ", sms通知:", smsErr)
models.DefaultAlarmManager.Create(uuid, models.AlarmTypeCPU, content, now)
}
}
}()
// 内存使用率
go func() {
windowTime := 5
ramThreshold := 10
ramCounter := 3
noticeWindowTime := 60
noticeCounter := int64(2)
for now := range time.Tick(time.Minute) {
beego.Debug("内存使用率告警", now)
startTime := now.Add(-1 * time.Duration(windowTime) * time.Minute) // 5 根据配置
noticeStartTime := now.Add(-1 * time.Duration(noticeWindowTime) * time.Minute) // 5 根据配置
var result []orm.Params
orm.NewOrm().Raw("SELECT uuid, count(*) as cnt from resource where deleted_time is null and created_time >= ? and ram_percent >= ? group by uuid having count(*) >= ?", startTime, ramThreshold, ramCounter).Values(&result)
for _, line := range result {
uuid, _ := line["uuid"].(string)
cntString, _ := line["cnt"].(string)
cnt, _ := strconv.Atoi(cntString)
content := fmt.Sprintf("终端[%s]在最近%d分钟内内存使用率大于%d%%的次数为%d, 已超过%d次", uuid, windowTime, ramThreshold, cnt, ramCounter)
alarmCnt := models.DefaultAlarmManager.GetCountByUuidAndType(uuid, models.AlarmTypeRam, noticeStartTime)
if alarmCnt >= noticeCounter {
beego.Info(fmt.Sprintf("通知次数(%d)超过限制(%d), %s", alarmCnt, noticeCounter, content))
continue
}
emailErr := emailSender.Send(to, "[CMDB]终端内存告警", content, []string{})
params := []string{uuid, strconv.Itoa(windowTime), strconv.Itoa(ramThreshold), strconv.Itoa(cnt), strconv.Itoa(ramCounter)}
smsErr := smsSender.Send(templateRamId, phones, params)
beego.Info("终端内存告警: ", content, ", email通知:", emailErr, ", sms通知:", smsErr)
models.DefaultAlarmManager.Create(uuid, models.AlarmTypeRam, content, now)
}
}
}()
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
}
D:\Workspace\Go\src\gocmdb\server\controllers\alarm.go
package controllers
import (
"strings"
"gocmdb/server/controllers/auth"
"gocmdb/server/models"
)
type AlarmPageController struct {
LayoutController
}
func (c *AlarmPageController) Index() {
c.Data["menu"] = "alarm_management"
c.Data["expand"] = "monitoring"
c.TplName = "alarm_page/index.html"
c.LayoutSections["LayoutScript"] = "alarm_page/index.script.html"
}
type AlarmController struct {
auth.LoginRequiredController
}
func (c *AlarmController) List() {
//draw,start, length, q
draw, _ := c.GetInt("draw")
start, _ := c.GetInt64("start")
length, _ := c.GetInt("length")
q := strings.TrimSpace(c.GetString("q"))
result, total, queryTotal := models.DefaultAlarmManager.Query(q, start, length)
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "获取成功",
"result": result,
"draw": draw,
"recordsTotal": total,
"recordsFiltered": queryTotal,
}
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\controllers\api\v2\api.go
func (c *APIController) Task() {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "成功",
"result": models.DefaultTaskManager.GetByUuid(c.Ctx.Input.Param(":uuid")),
}
c.ServeJSON()
}
func (c *APIController) TaskResult() {
rt := map[string]interface{}{
"code": 400,
"text": "",
"result": nil,
}
result := &models.Result{}
if err := json.Unmarshal(c.Ctx.Input.RequestBody, result); err == nil {
if err := models.DefaultTaskManager.Result(c.Ctx.Input.Param(":uuid"), result); err != nil {
rt["text"] = err.Error()
} else {
rt = map[string]interface{}{
"code": 200,
"text": "成功",
"result": nil,
}
}
} else {
rt["text"] = err.Error()
}
c.Data["json"] = rt
c.ServeJSON()
}
D:\Workspace\Go\src\gocmdb\server\controllers\layout.go
func (c *LayoutController) Prepare() {
alarmCount, alarms := models.DefaultAlarmManager.GetNotification(10)
c.Data["alarm"] = map[string]interface{}{
"count": alarmCount,
"list": alarms,
}
}D:\Workspace\Go\src\gocmdb\server\models\task.go
package models
import (
"fmt"
"time"
"github.com/astaxie/beego/orm"
)
type Task struct {
Id int `orm:"column(id);" json:"id"`
UUID string `orm:"column(uuid);size(64);" json:"uuid"`
Plugin string `orm:"column(plugin);size(32);" json:"plugin"`
Params string `orm:"column(params);type(text);" json:"params"`
Timeout int `orm:"column(timeout);" json:"timeout"`
Status int `orm:"column(status);" json:"status"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
CompletedTime *time.Time `orm:"column(completed_time);null;" json:"completed_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
Result *Result `orm:"column(result);reverse(one);" json:"result"`
}
type TaskManager struct{}
func NewTaskManager() *TaskManager {
return &TaskManager{}
}
func (m *TaskManager) Create(uuid string, plugin string, params string, timeout int) error {
ormer := orm.NewOrm()
task := &Task{
UUID: uuid,
Plugin: plugin,
Params: params,
Timeout: timeout,
Status: TaskStatusNew,
}
if _, err := ormer.Insert(task); err != nil {
return err
}
return nil
}
func (m *TaskManager) GetByUuid(uuid string) []*Task {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(new(Task))
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
condition = condition.And("uuid__exact", uuid)
condition = condition.And("status__in", TaskStatusNew)
var result []*Task
queryset.SetCond(condition).All(&result)
queryset.SetCond(condition).Update(orm.Params{"status": TaskStatusExecing})
return result
}
func (m *TaskManager) GetByIdAndUuid(id int, uuid string) *Task {
ormer := orm.NewOrm()
task := &Task{Id: id, UUID: uuid}
if err := ormer.Read(task, "id", "uuid"); err == nil {
return task
}
return nil
}
func (m *TaskManager) Result(uuid string, result *Result) error {
ormer := orm.NewOrm()
task := m.GetByIdAndUuid(result.TaskId, uuid)
if task == nil {
return fmt.Errorf("针对终端%s任务%s不存在", uuid, result.TaskId)
}
now := time.Now()
task.Status = TaskStatusSuccess
if result.Status != 0 {
task.Status = TaskStatusFailure
}
task.CompletedTime = &now
if _, err := ormer.Update(task); err != nil {
return err
}
result.Task = task
if _, err := ormer.Insert(result); err != nil {
return err
}
return nil
}
var DefaultTaskManager = NewTaskManager()
type Result struct {
Id int `orm:"column(id);" json:"id"`
Task *Task `orm:"column(task);rel(one);" json:"task"`
TaskId int `orm:"-" json:"task_id"`
Status int `orm:"-" json:"status"`
Result string `orm:"column(result);type(text);" json:"result"`
Err string `orm:"column(err);type(text);" json:"err"`
CreatedTime *time.Time `orm:"column(created_time);auto_now_add;" json:"created_time"`
DeletedTime *time.Time `orm:"column(deleted_time);null;" json:"deleted_time"`
}
func init() {
orm.RegisterModel(new(Task), new(Result))
}
D:\Workspace\Go\src\gocmdb\server\models\enum.go
const (
TaskStatusNew = iota
TaskStatusCancel
TaskStatusScheduling
TaskStatusExecing
TaskStatusSuccess
TaskStatusFailure
)
D:\Workspace\Go\src\gocmdb\server\models\alarm.go
func (m *AlarmManager) Query(q string, start int64, length int) ([]*Alarm, int64, int64) {
ormer := orm.NewOrm()
queryset := ormer.QueryTable(&Alarm{})
condition := orm.NewCondition()
condition = condition.And("deleted_time__isnull", true)
total, _ := queryset.SetCond(condition).Count()
qtotal := total
if q != "" {
query := orm.NewCondition()
condition = condition.AndCond(query)
qtotal, _ = queryset.SetCond(condition).Count()
}
var result []*Alarm
queryset.SetCond(condition).OrderBy("-created_time").Limit(length).Offset(start).All(&result)
return result, total, qtotal
}D:\Workspace\Go\src\gocmdb\server\controllers\api\v1\api.go
func (c *APIController) Task() {
c.Data["json"] = map[string]interface{}{
"code": 200,
"text": "成功",
"result": models.DefaultTaskManager.GetByUuid(c.Ctx.Input.Param(":uuid")),
}
c.ServeJSON()
}
func (c *APIController) TaskResult() {
rt := map[string]interface{}{
"code": 400,
"text": "",
"result": nil,
}
result := &models.Result{}
if err := json.Unmarshal(c.Ctx.Input.RequestBody, result); err == nil {
if err := models.DefaultTaskManager.Result(c.Ctx.Input.Param(":uuid"), result); err != nil {
rt["text"] = err.Error()
} else {
rt = map[string]interface{}{
"code": 200,
"text": "成功",
"result": nil,
}
}
} else {
rt["text"] = err.Error()
}
c.Data["json"] = rt
c.ServeJSON()
}D:\Workspace\Go\src\gocmdb\server\routers\router.go
// 告警页面
beego.AutoRouter(&controllers.AlarmPageController{})
// 告警
beego.AutoRouter(&controllers.AlarmController{})
v1Namespace := beego.NewNamespace("/v1",
beego.NSRouter("api/heartbeat/:uuid/", &v1.APIController{}, "*:Heartbeat"),
beego.NSRouter("api/register/:uuid/", &v1.APIController{}, "*:Register"),
beego.NSRouter("api/log/:uuid/", &v1.APIController{}, "*:Log"),
beego.NSRouter("api/task/:uuid/", &v1.APIController{}, "*:Task"),
beego.NSRouter("api/result/:uuid/", &v1.APIController{}, "*:TaskResult"),
)
beego.AddNamespace(v1Namespace)
v2Namespace := beego.NewNamespace("/v2",
beego.NSRouter("api/heartbeat/:uuid/", &v2.APIController{}, "*:Heartbeat"),
beego.NSRouter("api/register/:uuid/", &v2.APIController{}, "*:Register"),
beego.NSRouter("api/log/:uuid/", &v2.APIController{}, "*:Log"),
beego.NSRouter("api/task/:uuid/", &v2.APIController{}, "*:Task"),
beego.NSRouter("api/result/:uuid/", &v2.APIController{}, "*:TaskResult"),
)
beego.AddNamespace(v2Namespace)D:\Workspace\Go\src\gocmdb\server\views\alarm_page\index.html
D:\Workspace\Go\src\gocmdb\server\views\alarm_page\index.script.html
exhibit
Dashboard
terminal
Alarm management 
cloud platform added
Tencent Cloud
