java 线程池管理多线程操作Hbase数据库完整项目

在这里插入图片描述
Hbase-site.xml配置文件:

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hbase.rootdir</name>
        <value>hdfs://集群1(zhz100):9000/hbase</value>
    </property>
    <property>
        <name>hbase.cluster.distributed</name>
        <value>true</value>
    </property>
    <property>
        <name>hbase.master</name>
        <value>集群(zhz100):60000</value>
    </property>
    <property>
        <name>hbase.zookeeper.property.dataDir</name>
        <value>/usr/java/zookeeper3.4.10/temp</value>
    </property>
    <property>
        <name>hbase.zookeeper.quorum</name>
        <value>集群1(zhz100),集群2,集群3</value>
    </property>
    <property>
        <name>hbase.zookeeper.property.clientPort</name>
        <value>2181</value>
    </property>
</configuration>

配置application.yml:

	server:
	port: 8080
	
	hbase:
	    conf:
	        confMaps:
	        'hbase.zookeeper.quorum' : 'IP地址1:2181,IP地址2:2181'

pom.xml 引入的依赖:

   <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-logging</artifactId>
    </dependency>
    <dependency>
        <groupId>commons-io</groupId>
        <artifactId>commons-io</artifactId>
        <version>2.2</version>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-test</artifactId>
        <scope>test</scope>
    </dependency>

    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase-client</artifactId>
        <version>1.2.0</version>
        <exclusions>
            <exclusion>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    <dependency>
        <groupId>org.springframework.data</groupId>
        <artifactId>spring-data-hadoop</artifactId>
        <version>2.5.0.RELEASE</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.5.1</version>
    </dependency>
    <dependency>
        <groupId>org.springframework.data</groupId>
        <artifactId>spring-data-hadoop-core</artifactId>
        <version>2.4.0.RELEASE</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase</artifactId>
        <version>1.2.1</version>
        <type>pom</type>
    </dependency>
    <dependency>
        <groupId>ch.ethz.ganymed</groupId>
        <artifactId>ganymed-ssh2</artifactId>
        <version>build210</version>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
        <exclusions>
            <exclusion>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-logging</artifactId>
        </exclusion>
        </exclusions>
    </dependency>

Test含主函数main:

package com.example.demo;
import org.apache.hadoop.hbase.client.Connection;
import sun.net.ftp.FtpClient;
import sun.net.ftp.FtpProtocolException;

import java.io.*;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;


public class Test {
    FtpClient ftpClient;

/**
 * 连接FTP服务
 * @param url //IP地址
 * @param port//端口号
 * @param username//用户名
 * @param password//密码
 * @return
 */
public static FtpClient connectFTP(String url, int port, String username, String password) {

         //创建ftp
     FtpClient ftp = null;
    try {
        //创建地址
        SocketAddress addr = new InetSocketAddress(url, port);
        //连接
        ftp = FtpClient.create();
        ftp.connect(addr);
        //登陆
        ftp.login(username, password.toCharArray());
        ftp.setBinaryType();

    } catch (FtpProtocolException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return ftp;
}

public static List<String> download(String ftpFile, FtpClient ftp) {
    synchronized (TimerWatch.class) {
        List<String> list = new ArrayList<String>();
        String str = "";
        InputStream is = null;
        BufferedReader br = null;
        try {
            // 获取ftp上的文件
            is = ftp.getFileStream(ftpFile);
            //转为字节流
            br = new BufferedReader(new InputStreamReader(is));
            while ((str = br.readLine()) != null) {
                list.add(str);
            }
            br.close();
        } catch (FtpProtocolException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }
}
    public static void main(String[] args) throws Exception{

	//开启主机1FTP服务器
        FtpClient ftp = connectFTP("主机1IP"端口号,"账号","密码");
	//开启主机1Linux指令
        RemoteExecuteCommand rec = new RemoteExecuteCommand("主机1IP", "账号", "密码"); 
        //连接数据库
        Connection connection=LinkHbase.table();
	//如果出现宕机或者出现异常导致项目停止运行,此段代码可以将未成功插入的数据所在的文件就会保存在work目录中。重启后对work进行解析就会避免数据丢失,此方法只会在启动时执行一次。            
         Sync.Judge(connection,ftp,rec);
        //开启一个线程池,corePoolSize:核心线程数5;maxPoolSize:最大线程 数10; keepAliveTime:线程空闲时间:200微妙;ArrayBlockingQueue是一个基于数组结构的有界阻塞队列,此队列按 FIFO(先进先出)原则对元素进行排序
        ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 10, 200,                 TimeUnit.MILLISECONDS,new ArrayBlockingQueue<Runnable>(5));

        for(int i=0;i<5;i++){
            MyTask myTask = new MyTask(i,connection,ftp,rec);
            executor.execute(myTask);
            System.out.println("线程池中线程数目:"+executor.getPoolSize()+",队列中等待执行的任务数目:"+
                    executor.getQueue().size()+",已执行玩别的任务数目:"+executor.getCompletedTaskCount());
        }
        executor.shutdown();
    }
}

Sync类中Judge()方法:

//判断in文件是否有文件执行完成
public static void Judge(Connection connection,FtpClient ftp,RemoteExecuteCommand rec) throws Exception {
	
//Linux命令获取/usr/HDWork/work的文件名字
   String files=rec.execute("cd /usr/HDWork/work ; ls -lrt|sed -n '2, $p'|awk '{print $9}'");
    if(files.equals("") || files==null){
        System.out.println("work目录下没有文件");
    } else {
        System.out.println("work目录有文件");
        String[] strs = files.split("\\n");
        for (int i = 0, len = strs.length; i < len; i++) {
            SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//计算运行时间
            long start, end;
            start = System.currentTimeMillis();
            Date date1 = new Date(start);
            String date1String = formatter.format(date1);

            System.out.println(strs[i].toString());
	//解析work目录下的文件内容,插入到hbase
            ReadIn.add(strs[i].toString(), connection, ftp);                
        //移动文件到back目录下
        Cmd.mvBack(strs[i].toString(), rec);
            end = System.currentTimeMillis();
            Date date2 = new Date(end);
            String date2String = formatter.format(date2);
            System.out.println(Thread.currentThread().getName() + ":执行文件:." + strs[i].toString() + ";开始时间为:" + date1String + ";结束时间为:" + date2String + ";添加成功,共用时" + (end - start)+ "ms。");
                }
    }

ReadIn类中add()方法:

public class ReadIn {
    /*
    *解析work目录下的文件内容,插入到数据库中
    * */
    public static void add(String fileName, Connection connection, FtpClient ftpClient) throws Exception{
    String[] y= { "trademark" , "dr_type" , "service_id" , "bill_month" );
    Table table=table(connection);//判断是否已经存在含有这个表
    List<String> cutstrings;
    System.out.println("/usr/HDWork/work/"+fileName);
    List<String> list=download("/usr/HDWork/work/"+fileName,ftpClient);

    String str;

            for(int i=6;i<list.size();i++){
                int aa = (int) ((Math.random() * 9 + 1) * 100000);
                System.out.println(aa + "根据文件名来得到keyRow"+fileName);
                Put put = new Put(Bytes.toBytes( aa));

                cutstrings = cutstring(list.get(i).substring(19));
                for (int z = 0; z < cutstrings.size() - 1; z++) {  // 每一个for读入一行数据
                    put.addColumn(Bytes.toBytes(y[z]), Bytes.toBytes("aa"), Bytes.toBytes(cutstrings.get(z)));//将字段与值匹配对应插入
                }
                table.put(put);//提交数据
            }
}

/*
* 线程锁判断是否有这个表
* */

public static Table table(Connection connection)throws Exception{
    synchronized (ReadIn.class) {
        Admin admin = connection.getAdmin();
        Table table = null;
        SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");
        String systemTime = String.valueOf(df.format(new Date()));//系统日期
        TableName tableNameObj = TableName.valueOf(systemTime);//以系统日期为表的名字
        if (admin.tableExists(tableNameObj)) {//如果表的名字已经存在
            table = connection.getTable(TableName.valueOf(systemTime));//连接这个表
            return table;
        } else {//没有这个表,先创建表,在链接表
            HbaseDemo.createTable(systemTime, "trademark", "dr_type", "service_id");
            table = connection.getTable(TableName.valueOf(systemTime));
            return table;
        }
    }
}

/*
 * 截取前19位字符后解析的内容
 * */
public   static List<String> cutstring (String Stence) {
    List<String> stringlist = new ArrayList<String>();//用来存储解析出来的元素
    for (int i = 0; i < Stence.length(); i++) {
        if (Stence.charAt(i) == ';') {
            String temp = "";//存储单词
            int wordlength = i;
            while (wordlength < Stence.length() - 1 && Stence.charAt(++wordlength) != ';') {
                temp += Stence.charAt(wordlength);
                //System.out.println(temp);
            }
            stringlist.add(temp);
        }
    }
    return stringlist;
}
/**
 * 取ftp上的文件内容
 * @param ftpFile
 * @param ftp
 * @return
 */
public static List<String> download(String ftpFile, FtpClient ftp) {
    synchronized (ReadIn.class) {
        List<String> list = new ArrayList<String>();
        String str = "";
        InputStream is = null;
        BufferedReader br = null;
        try {
            ftpFile=ftpFile.replace(" ","");
            // 获取ftp上的文件
            is = ftp.getFileStream(ftpFile);
            //转为字节流
            br = new BufferedReader(new InputStreamReader(is));
            while((str=br.readLine())!=null){
                list.add(str);
            }
            System.out.println(list+"-------------");
            br.close();
        }catch (FtpProtocolException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }
}

}

MyTask类:

package com.example.demo;

import org.apache.hadoop.hbase.client.Connection;
import sun.applet.Main;
import sun.net.ftp.FtpClient;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.locks.Lock;
import java.util.logging.FileHandler;
import java.util.logging.Logger;

public class MyTask implements Runnable {
    private int taskNum;
    private Connection connection;
    private Lock lock;
    private FtpClient ftpClient;
    private RemoteExecuteCommand rec;



    public MyTask(int num, Connection connection, FtpClient ftpClient,RemoteExecuteCommand rec) {
        this.taskNum = num;
        this.connection=connection;
        this.ftpClient=ftpClient;
        this.rec=rec;
    }
    
    @Override
    public void run(){

        System.out.println(Thread.currentThread().getName() + "线程的名字");
        final Logger logger=Logger.getLogger(Main.class.toString());//日志
        StringBuffer logPath=new StringBuffer();

        logPath.append("E:\\Logger");     //设置保存路径

        //设置文件名
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        logPath.append("\\"+sdf.format(new Date())+".log");
        //将输出handler加入logger
        try {
            FileHandler fileHandler=new FileHandler(logPath.toString(),true);
            logger.addHandler(fileHandler);
        }catch (IOException e){
            e.printStackTrace();
        }

        try {

            while (true) {
                SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                long start,end;
                start = System.currentTimeMillis();
                Date date1=new Date(start);
                String date1String = formatter.format(date1);


                RemoteExecuteCommand rec=new RemoteExecuteCommand("10.248.56.120", "root","password");
               String aa= rec.execute("ls /usr/HDWork/in");
                if (aa.length() > 0) {


                    //Sync sync = new Sync();//开启静态锁,(静态锁不能封装到方法中,此处尝试错误)
                  // String fileName=sync.Mv();//调用静态锁方法,获取work目录中文件时间最小的文件,将文件名称存放在1.txt;将获取的文件移动到in目录中等待操作,删除1.txt。
                    String fileName=this.Mv();//静态锁方法,获取work目录中文件时间最小的文件名字,将文件名称存放在1.txt;将获取的文件移动到in目录中等待操作,将文件名字返回,。
                    if(fileName==null || fileName.equals("")){
                        System.out.println(Thread.currentThread().getName()+"没有文件");
                    }else {
                        ReadIn.add(fileName, connection,ftpClient);//解析in目录下的文件内容,插入到hbase
                    }
                    if(fileName==null || fileName.equals("")){
                        System.out.println(Thread.currentThread().getName()+"没有文件");
                    }else {
                       if(Cmd.mvBack(fileName,rec)) {
                           end = System.currentTimeMillis();
                           Date date2=new Date(end);
                           String date2String = formatter.format(date2);
                           System.out.println(Thread.currentThread().getName() + ":执行文件:." + fileName + ";开始时间为:"+date1String+";结束时间为:"+date2String+";添加成功,共用时"+(end-start)+"ms。");
                           logger.info(Thread.currentThread().getName() + ":执行文件:." + fileName + ";开始时间为:"+date1String+";结束时间为:"+date2String+";添加成功,共用时"+(end-start)+"ms。");
                        }else {
                            System.out.println("过滤异常");
                            logger.info("数据插入成功,但是移动到back中失败");
                        }
                    }
                } else {

                    System.out.println("run中的=="+Thread.currentThread().getName()+"没有文件");
                    Thread.sleep(1000);
                }
            }
        }catch (Exception e){

            System.out.println("run异常"+e.getMessage());
        }
    }

//此处用到了线程锁,避免多个线程同时操作同一个文件,重复添加
    synchronized public String  Mv(){
        synchronized (MyTask.class) {
            rec.execute("cd /usr/HDWork/in");
            String fileName = rec.execute("cd /usr/HDWork/in ; ls -lrt|sed -n \"2, 1p\"|awk '{print $9}'");
            fileName = fileName.replace("\n", " ");
            String mv = "mv /usr/HDWork/in/" + fileName + " /usr/HDWork/work";
            if (!"".equals(fileName)) {
            rec.execute(mv);
            System.out.println(fileName + "======================");
            return fileName;
        } else {
            return null;
        }
    }
    }
}

LinkHbase类:连接数据库

public class LinkHbase {
    private static Configuration conf = HBaseConfiguration.create();
    private static Admin admin;
    static {
        conf.set("hbase.rootdir", "hdfs://node1:9000/hbase");
        // 设置Zookeeper,直接设置IP地址
        conf.set("hbase.zookeeper.quorum", "集群1,集群2,集群3");
    }
    public static Connection table() throws  Exception{
        Connection connection = ConnectionFactory.createConnection(conf);
        return  connection;
    }
}

RemoteExcuteCommaand类:远程执行Linux

/**
 * 远程执行linux的shell script
 * @author Ickes
 * @since  V0.1
 */		
public class RemoteExecuteCommand {

    //字符编码默认是utf-8
    private static String  DEFAULTCHART="UTF-8";
    private Connection conn;
    private String ip;
    private String userName;
    private String userPwd;

    public RemoteExecuteCommand(String ip, String userName, String userPwd) {
        this.ip = ip;
        this.userName = userName;
        this.userPwd = userPwd;
    }
   //无参方法
    public RemoteExecuteCommand() {
    }
    /**
     * 远程登录linux的主机
     * @author Ickes
     * @since  V0.1
     * @return
     *      登录成功返回true,否则返回false
     */
    public Boolean login(){
        boolean flg=false;
        try {
            conn = new Connection(ip);
            conn.connect();//连接
            flg=conn.authenticateWithPassword(userName, userPwd);//认证
        } catch (IOException e) {
            e.printStackTrace();
        }
        return flg;
    }

/**
 * @author Ickes
 * 远程执行shll脚本或者命令
 * @param cmd
 *      即将执行的命令
 * @return
 *      命令执行完后返回的结果值
 * @since V0.1
 */
public String execute(String cmd){
    String result="";
    try {
        if(login()){
            Session session= conn.openSession();//打开一个会话
            session.execCommand(cmd);//执行命令
            result=processStdout(session.getStdout(),DEFAULTCHART);
            //如果为得到标准输出为空,说明脚本执行出错了
            if(StringUtils.isBlank(result)){
                result=processStdout(session.getStderr(),DEFAULTCHART);
            }
            conn.close();
            session.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return result;
}

/**
 * @author Ickes
 * 远程执行shll脚本或者命令
 * @param cmd
 *      即将执行的命令
 * @return
 *      命令执行成功后返回的结果值,如果命令执行失败,返回空字符串,不是null
 * @since V0.1
 */
public String executeSuccess(String cmd){
    String result="";
    try {
        if(login()){
            Session session= conn.openSession();//打开一个会话
            session.execCommand(cmd);//执行命令
            result=processStdout(session.getStdout(),DEFAULTCHART);
            conn.close();
            session.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return result;
}

/**
 * 解析脚本执行返回的结果集
 * @author Ickes
 * @param in 输入流对象
 * @param charset 编码
 * @since V0.1
 * @return
 *       以纯文本的格式返回
 */

private String processStdout(InputStream in, String charset){
    InputStream    stdout = new StreamGobbler(in);
    StringBuffer buffer = new StringBuffer();;
    try {
        BufferedReader br = new BufferedReader(new InputStreamReader(stdout,charset));
        String line=null;
        while((line=br.readLine()) != null){
            buffer.append(line+"\n");
        }
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    } catch (IOException e) 

        e.printStackTrace();
    }
    return buffer.toString();
}

public static void setCharset(String charset) {
    DEFAULTCHART = charset;
}
public Connection getConn() {
    return conn;
}
public void setConn(Connection conn) {
    this.conn = conn;
}
public String getIp() {
    return ip;
}
public void setIp(String ip) {
    this.ip = ip;
}
public String getUserName() {
    return userName;
}
public void setUserName(String userName) {
    this.userName = userName;
}
public String getUserPwd() {
    return userPwd;
}
public void setUserPwd(String userPwd) {
    this.userPwd = userPwd;
}

猜你喜欢

转载自blog.csdn.net/qq_36484611/article/details/86063454