数据库连接线程池批处理

DB>
ConnectionPool.java
package com.citi.isg.smc.mw.delta.db;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;

import oracle.jdbc.pool.OracleDataSource;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.citi.isg.smc.mw.delta.utils.PropertyConfig;

public class ConnectionPool {
private static Log log = LogFactory.getLog(ConnectionPool.class);
private BlockingQueue<PooledConnection> reserved;
private Collection<PooledConnection> allocated;
private int createdConnections = 0;

private OracleDataSource ds;

public ConnectionPool(int capacity) {
try {
ds = new OracleDataSource();
ds.setURL(PropertyConfig.getConfiguration().getString(
PropertyConfig.DB_URL));
ds.setUser(PropertyConfig.getConfiguration().getString(
PropertyConfig.DB_USER));
ds.setPassword(PropertyConfig.getConfiguration().getString(
PropertyConfig.DB_PASSWORD));
} catch (SQLException e) {
log.error("SQLException: ", e);
}
reserved = new ArrayBlockingQueue<PooledConnection>(capacity);
allocated = new ArrayList<PooledConnection>();
}

public PooledConnection getConnection() throws SQLException {
PooledConnection pc = reserved.poll();
if (pc == null) {
int currentRetry = 0;
while (currentRetry < 5) {
try {
Connection conn = ds.getConnection();
createdConnections++;
log.debug("Create a new connection named C-"
+ createdConnections + ".");
pc = new PooledConnection(conn, "C-" + createdConnections);
allocated.add(pc);
return pc;
} catch (SQLException e) {
log.error("SQLException: ", e);
currentRetry++;
if (currentRetry > 5) {
log.error("Get connection failed after all retries", e);
throw new SQLException(
"Get connection failed after all retries");
} else {
if (currentRetry > 0) {
log.warn("Get connection failed" + currentRetry
+ " times ", e);
}
try {
Thread.sleep(15000);
} catch (Exception ex) {
log.error("Exception: ", ex);
}
}
}
}
throw new SQLException("Get connection failed after all retries");
} else {
log.debug("Find a connection from pool with name " + pc.getName());
return pc;
}
}

public void releaseConnection(PooledConnection pc) {
if (pc == null)
return;

allocated.remove(pc);
try {
if (pc.getConnection() != null && !pc.getConnection().isClosed()) {
try {
reserved.add(pc);
} catch (IllegalStateException ise) {
log.debug("reserved queue is full!");
pc.getConnection().close();
}
}
} catch (SQLException e) {
log.error("SQLException: ", e);
}
}

public void resetConnection(PooledConnection pc) {
String oldName = pc.getName();
try {
if (pc != null && pc.getConnection() != null) {
pc.getConnection().close();
allocated.remove(pc);
}
pc = getConnection();
} catch (SQLException e) {
log.error("SQLException: ", e);
}
log.debug("Replace pooledConnection " + oldName + " with "
+ pc.getName());
}

public void close() {
for (PooledConnection pc : reserved) {
if (pc != null && pc.getConnection() != null) {
try {
pc.getConnection().close();
} catch (SQLException e) {
log.error("SQLException: ", e);
}
}
}
if (!allocated.isEmpty()) {
StringBuffer sb = new StringBuffer("Still have allocated connections");
for (PooledConnection pc : allocated) {
sb.append(" "+pc.getName()+",");
if (pc != null && pc.getConnection() != null) {
try {
pc.getConnection().close();
} catch (SQLException e) {
log.error("SQLException: ", e);
}
}

}
log.debug(sb.toString());
}
}
}

>DbmsOutput.java
package com.citi.isg.smc.mw.delta.db;

import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.SQLException;

public class DbmsOutput {
    /*
     * our instance variables. It is always best to use callable or prepared
     * statements and prepare (parse) them once per program execution, rather
     * then one per execution in the program. The cost of reparsing is very
     * high. Also -- make sure to use BIND VARIABLES!
     *
     * we use three statments in this class. One to enable dbms_output -
     * equivalent to SET SERVEROUTPUT on in SQL*PLUS. another to disable it --
     * like SET SERVEROUTPUT OFF. the last is to "dump" or display the results
     * from dbms_output using system.out
     *
     */
    private CallableStatement enable_stmt;
    private CallableStatement disable_stmt;
    private CallableStatement show_stmt;

    /*
     * our constructor simply prepares the three statements we plan on
     * executing.
     *
     * the statement we prepare for SHOW is a block of code to return a String
     * of dbms_output output. Normally, you might bind to a PLSQL table type but
     * the jdbc drivers don't support PLSQL table types -- hence we get the
     * output and concatenate it into a string. We will retrieve at least one
     * line of output -- so we may exceed your MAXBYTES parameter below. If you
     * set MAXBYTES to 10 and the first line is 100 bytes long, you will get the
     * 100 bytes. MAXBYTES will stop us from getting yet another line but it
     * will not chunk up a line.
     *
     */
    public DbmsOutput(Connection conn) throws SQLException {
        enable_stmt = conn.prepareCall("begin dbms_output.enable(:1); end;");
        disable_stmt = conn.prepareCall("begin dbms_output.disable; end;");

        show_stmt = conn.prepareCall("declare " + "    l_line varchar2(255); " + "    l_done number; " + "    l_buffer long; " + "begin " + " loop " + "    exit when length(l_buffer)+255 > :maxbytes OR l_done = 1; " + "    dbms_output.get_line( l_line, l_done ); " + "    l_buffer := l_buffer || l_line || chr(10); " + " end loop; " + " :done := l_done; " + " :buffer := l_buffer; " + "end;");
    }

    /*
     * enable simply sets your size and executes the dbms_output.enable call
     *
     */
    public void enable(int size) throws SQLException {
        enable_stmt.setInt(1, size);
        enable_stmt.executeUpdate();
    }

    /*
     * disable only has to execute the dbms_output.disable call
     */
    public void disable() throws SQLException {
        disable_stmt.executeUpdate();
    }

    /*
     * show does most of the work. It loops over all of the dbms_output data,
     * fetching it in this case 32,000 bytes at a time (give or take 255 bytes).
     * It will print this output on stdout by default (just reset what
     * System.out is to change or redirect this output).
     */

    public String show() throws SQLException {
        int done = 0;
        StringBuffer retStr = new StringBuffer();

        show_stmt.registerOutParameter(2, java.sql.Types.INTEGER);
        show_stmt.registerOutParameter(3, java.sql.Types.VARCHAR);

        for (;;) {
            show_stmt.setInt(1, 32000);
            show_stmt.executeUpdate();
            retStr.append(show_stmt.getString(3));
            if ((done = show_stmt.getInt(2)) == 1)
                break;
        }
        return retStr.toString();
    }

    /*
     * close closes the callable statements associated with the DbmsOutput
     * class. Call this if you allocate a DbmsOutput statement on the stack and
     * it is going to go out of scope -- just as you would with any callable
     * statement, result set and so on.
     */
    public void close() throws SQLException {
        enable_stmt.close();
        disable_stmt.close();
        show_stmt.close();
    }
}
>PooledConnection.java
package com.citi.isg.smc.mw.delta.db;

import java.sql.Connection;

public class PooledConnection {
private Connection connection;
private String name;

public PooledConnection(Connection connection, String name) {
this.connection = connection;
this.name = name;
}

public Connection getConnection() {
return connection;
}

public void setConnection(Connection connection) {
this.connection = connection;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

}
model.
>DataObject.java
package com.citi.isg.smc.mw.delta.model;

public interface DataObject {
public String getData();
}

>DeltaData.java
package com.citi.isg.smc.mw.delta.model;

import java.util.Date;

public class DeltaData implements DataObject {
private String smcp;
private int version;
private String type;
private String data;

private Date modTs;

public DeltaData() {
}

public DeltaData(String smcp, int version, String type, String data,
Date modTs) {
this.smcp = smcp;
this.version = version;
this.type = type;
this.data = data;
this.modTs = modTs;
}

public String getSmcp() {
return smcp;
}

public void setSmcp(String smcp) {
this.smcp = smcp;
}

public int getVersion() {
return version;
}

public void setVersion(int version) {
this.version = version;
}

public String getType() {
return type;
}

public void setType(String type) {
this.type = type;
}

public String getData() {
return data;
}

public void setData(String data) {
this.data = data;
}

public Date getModTs() {
return modTs;
}

public void setModTs(Date modTs) {
this.modTs = modTs;
}

}

>FixedSizeList.java
package com.citi.isg.smc.mw.delta.model;

import java.util.ArrayList;

public class FixedSizeList<E extends DataObject> extends ArrayList<E> {
private static final long serialVersionUID = 1L;
private int maxItemSize;
private int maxDataSize;
private int dataSize;

public FixedSizeList(int maxItemSize, int maxDataSize) {
this.maxItemSize = maxItemSize;
this.maxDataSize = maxDataSize;
}

public int getMaxItemSize() {
return maxItemSize;
}

public boolean add(E e) {
if (isFull()) {
return false;
} else {
if (e.getData() == null) {
return false;
} else {
dataSize += e.getData().length();
return super.add(e);
}
}
}

public void setMaxItemSize(int maxItemSize) {
this.maxItemSize = maxItemSize;
}

public boolean isFull() {
if (maxItemSize != -1 && this.size() >= this.getMaxItemSize()) {
return true;
}
if (maxDataSize != -1 && this.getDataSize() >= this.getMaxDataSize()) {
return true;
}
return false;
}

public int getMaxDataSize() {
return maxDataSize;
}

public void setMaxDataSize(int maxDataSize) {
this.maxDataSize = maxDataSize;
}

public int getDataSize() {
return dataSize;
}

public void setDataSize(int dataSize) {
this.dataSize = dataSize;
}
}
>ServiceQueue.java
package com.citi.isg.smc.mw.delta.model;

import java.util.concurrent.LinkedBlockingDeque;

public class ServiceQueue<E extends DataObject> {

private LinkedBlockingDeque<FixedSizeList<E>> queue;
private int maxItemSize;
private int maxDataSize;

public ServiceQueue(int queueSize, int maxItemSize, int maxDataSize) {
this.maxItemSize = maxItemSize;
this.maxDataSize = maxDataSize;
this.queue = new LinkedBlockingDeque<FixedSizeList<E>>(queueSize);
}

public boolean batchPrepared() {
return queue.isEmpty() ? false : queue.getFirst().isFull();
}

public void add(E e) throws InterruptedException {
if (queue.isEmpty() || queue.getLast().isFull()) {
queue.put(new FixedSizeList<E>(maxItemSize, maxDataSize));
}
queue.getLast().add(e);
}

public FixedSizeList<E> getBatch() {
return queue.poll();
}

public int size() {
return queue.size();
}
}

.parser
DeltaXMLParser.java
package com.citi.isg.smc.mw.delta.parser;

import java.io.IOException;
import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.Date;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

public class DeltaXMLParser {
private static Log log = LogFactory
.getLog(DeltaXMLParser.class);

private Document doc;
private XPath xpath;
private final static SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmssSSS");

public DeltaXMLParser(String deltaStr) {
try {

DocumentBuilderFactory factory = DocumentBuilderFactory
.newInstance();
factory.setNamespaceAware(true);
DocumentBuilder builder = factory.newDocumentBuilder();
this.doc = builder
.parse(new InputSource(new StringReader(deltaStr)));
XPathFactory xpFactory = XPathFactory.newInstance();
this.xpath = xpFactory.newXPath();
} catch (SAXException e) {
log.debug("SAXException : "+e.toString());
} catch (ParserConfigurationException e) {
log.debug("ParserConfigurationException : "+e.toString());

} catch (IOException e) {
log.debug("IOException : "+e.toString());

}
}

public String getSmcp() throws Exception {
XPathExpression expr = xpath.compile("/Delta/CurrentAsset/@id");

String result = expr.evaluate(doc, XPathConstants.STRING).toString();
return result;

}

public String getType() throws Exception {
XPathExpression expr = xpath.compile("/Delta/CurrentAsset/@type");

String result = expr.evaluate(doc, XPathConstants.STRING).toString();
return result;

}

public int getVersion() throws Exception {
XPathExpression expr = xpath.compile("/Delta/CurrentAsset/@version");

String result = expr.evaluate(doc, XPathConstants.STRING).toString();
return Integer.parseInt(result);
}

public Date getModTs() throws Exception {
XPathExpression expr = xpath.compile("/Delta/CurrentAsset/@modified");

String result = expr.evaluate(doc, XPathConstants.STRING).toString();
return format.parse(result);
}

public static void main(String[] args) throws Exception {
String deltaStr = "<Delta><CurrentAsset id=\"16426709\" code=\"SMCP\" type=\"OPTION\" version=\"10\" modified=\"20120928185223638\"/><OldAsset id=\"16426709\" code=\"SMCP\" type=\"OPTION\" version=\"9\" modified=\"20120610001356554\"/><Delete><concepts><concept cname=\"XrefHistory\" cversion=\"9\"><recs><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20110713014853754</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101214024745895</attribute><attribute name=\"XrefValue\" key=\"true\">OKSS161109500.Y</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ISN</attribute><attribute name=\"XrefEndDate\">20120610001356404</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501955</attribute><attribute name=\"XrefValue\" key=\"true\">DE000P0GCC83</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20110719070355795</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708730</attribute><attribute name=\"XrefValue\" key=\"true\">OKS</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ETS</attribute><attribute name=\"XrefEndDate\">20110713014853767</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708663</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ETS</attribute><attribute name=\"XrefEndDate\">20110719070355792</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110713014853736</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00047500</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20120610001356410</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501956</attribute><attribute name=\"XrefValue\" key=\"true\">OSFX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20110719070355732</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110714020205918</attribute><attribute name=\"XrefValue\" key=\"true\">OKSS161104750.Y</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110713014853772</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708789</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000XISX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20120610001356396</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501953</attribute><attribute name=\"XrefValue\" key=\"true\">OSFX310X2.EX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110719070355788</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110713014853729</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00047500XISX</attribute></attributes></rec></recs></concept></concepts></Delete><Modify><concepts><concept cname=\"ContractInformation\" cversion=\"10\"><recs><rec><attributes><attribute name=\"StrikePriceMajor\">31.0</attribute></attributes></rec></recs></concept><concept cname=\"PriceXref\" cversion=\"10\"><recs><rec><attributes><attribute name=\"EndDate\">20120609223249000</attribute><attribute name=\"PriceTag\" key=\"true\">RT:OSFX310X2.EX</attribute><attribute name=\"StartDate\" key=\"true\">20111121221855000</attribute></attributes></rec></recs></concept><concept cname=\"Product\" cversion=\"10\"><recs><rec><attributes><attribute name=\"Description\">OSFX DEC2 31 P</attribute></attributes></rec></recs></concept></concepts></Modify><Add><concepts><concept cname=\"IssueClassification\" cversion=\"10\"><recs><rec><attributes><attribute name=\"RiskSecTypeLevel1\">OPTION</attribute><attribute name=\"RiskSecTypeLevel2\">COMMODITY-OPTION</attribute></attributes></rec></recs></concept><concept cname=\"RiskInformation\" cversion=\"10\"><recs><rec><attributes><attribute name=\"Granularity\">99</attribute></attributes></rec></recs></concept><concept cname=\"XrefHistory\" cversion=\"10\"><recs><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20110719070355795</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110713014853772</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000XISX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20120610001356410</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501000</attribute><attribute name=\"XrefValue\" key=\"true\">OSFX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20110719070355732</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110714020205000</attribute><attribute name=\"XrefValue\" key=\"true\">OKSS161104750.Y</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110719070355788</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110713014853000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00047500XISX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ETS</attribute><attribute name=\"XrefEndDate\">20110713014853767</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\"";
DeltaXMLParser parser = new DeltaXMLParser(deltaStr);
System.out.println(parser.getSmcp());
System.out.println(parser.getType());
System.out.println(parser.getVersion());
}
}

>QuickDeltaXMLParser.java
package com.citi.isg.smc.mw.delta.parser;

import java.text.SimpleDateFormat;
import java.util.Date;

public class QuickDeltaXMLParser {
private String smcp;
private String type;
private int version;
private Date modTs;
private final static SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmssSSS");

public QuickDeltaXMLParser(String deltaStr) throws Exception {
String truncatePart = deltaStr.substring(0, deltaStr.indexOf("/><OldAsset"));
String[] values = truncatePart.split(" ");
for (String value : values) {
if (value.startsWith("id=")) {
smcp = getValue(value);
}
if (value.startsWith("type=")) {
type = getValue(value);
}
if (value.startsWith("version=")) {
version = Integer.parseInt(getValue(value));
}
if (value.startsWith("modified")) {
modTs = format.parse(getValue(value));
}
}
}

public static String getValue(String str) {
String[] strs = str.split("\"");
return strs[1];
}

public String getSmcp() {
return smcp;
}

public String getType() {
return type;
}

public int getVersion() {
return version;
}

public Date getModTs() {
return modTs;
}

public static void main(String[] args) throws Exception {
String deltaStr = "<Delta><CurrentAsset id=\"16426709\" code=\"SMCP\" type=\"OPTION\" version=\"10\" modified=\"20120928185223638\"/><OldAsset id=\"16426709\" code=\"SMCP\" type=\"OPTION\" version=\"9\" modified=\"20120610001356554\"/><Delete><concepts><concept cname=\"XrefHistory\" cversion=\"9\"><recs><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20110713014853754</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101214024745895</attribute><attribute name=\"XrefValue\" key=\"true\">OKSS161109500.Y</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ISN</attribute><attribute name=\"XrefEndDate\">20120610001356404</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501955</attribute><attribute name=\"XrefValue\" key=\"true\">DE000P0GCC83</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20110719070355795</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708730</attribute><attribute name=\"XrefValue\" key=\"true\">OKS</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ETS</attribute><attribute name=\"XrefEndDate\">20110713014853767</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708663</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ETS</attribute><attribute name=\"XrefEndDate\">20110719070355792</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110713014853736</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00047500</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20120610001356410</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501956</attribute><attribute name=\"XrefValue\" key=\"true\">OSFX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20110719070355732</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110714020205918</attribute><attribute name=\"XrefValue\" key=\"true\">OKSS161104750.Y</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110713014853772</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708789</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000XISX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20120610001356396</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501953</attribute><attribute name=\"XrefValue\" key=\"true\">OSFX310X2.EX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110719070355788</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110713014853729</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00047500XISX</attribute></attributes></rec></recs></concept></concepts></Delete><Modify><concepts><concept cname=\"ContractInformation\" cversion=\"10\"><recs><rec><attributes><attribute name=\"StrikePriceMajor\">31.0</attribute></attributes></rec></recs></concept><concept cname=\"PriceXref\" cversion=\"10\"><recs><rec><attributes><attribute name=\"EndDate\">20120609223249000</attribute><attribute name=\"PriceTag\" key=\"true\">RT:OSFX310X2.EX</attribute><attribute name=\"StartDate\" key=\"true\">20111121221855000</attribute></attributes></rec></recs></concept><concept cname=\"Product\" cversion=\"10\"><recs><rec><attributes><attribute name=\"Description\">OSFX DEC2 31 P</attribute></attributes></rec></recs></concept></concepts></Modify><Add><concepts><concept cname=\"IssueClassification\" cversion=\"10\"><recs><rec><attributes><attribute name=\"RiskSecTypeLevel1\">OPTION</attribute><attribute name=\"RiskSecTypeLevel2\">COMMODITY-OPTION</attribute></attributes></rec></recs></concept><concept cname=\"RiskInformation\" cversion=\"10\"><recs><rec><attributes><attribute name=\"Granularity\">99</attribute></attributes></rec></recs></concept><concept cname=\"XrefHistory\" cversion=\"10\"><recs><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20110719070355795</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110713014853772</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000XISX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">TCK</attribute><attribute name=\"XrefEndDate\">20120610001356410</attribute><attribute name=\"XrefStartDate\" key=\"true\">20111122061501000</attribute><attribute name=\"XrefValue\" key=\"true\">OSFX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">RIC</attribute><attribute name=\"XrefEndDate\">20110719070355732</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110714020205000</attribute><attribute name=\"XrefValue\" key=\"true\">OKSS161104750.Y</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">OSIK</attribute><attribute name=\"XrefEndDate\">20110719070355788</attribute><attribute name=\"XrefStartDate\" key=\"true\">20110713014853000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00047500XISX</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\" key=\"true\">ETS</attribute><attribute name=\"XrefEndDate\">20110713014853767</attribute><attribute name=\"XrefStartDate\" key=\"true\">20101211030708000</attribute><attribute name=\"XrefValue\" key=\"true\">OKS   110716P00095000</attribute></attributes></rec><rec><attributes><attribute name=\"XrefCode\"";
QuickDeltaXMLParser parser = new QuickDeltaXMLParser(deltaStr);
System.out.println(parser.getSmcp());
System.out.println(parser.getType());
System.out.println(parser.getVersion());
System.out.println(parser.getModTs());
}
}

utils
PropertyConfig.java
package com.citi.isg.smc.mw.delta.utils;

import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.citi.isg.smc.mw.delta.parser.DeltaXMLParser;

public class PropertyConfig {
private static Log log = LogFactory
.getLog(PropertyConfig.class);
private static PropertiesConfiguration configation;
public static String DB_DRIVER = "db.smc.driver";
public static String DB_URL = "db.smc.url";
public static String DB_USER = "db.smc.user";
public static String DB_PASSWORD = "db.smc.password";
public static String POOL_SIZE = "db.smc.connectionpool.size";

public static String MAX_ITEM_SIZE = "delta.loader.batch.max_item";
public static String MAX_DATA_SIZE = "delta.loader.batch.max_total_data_size";
public static String DEADLOCK_RETRY = "delta.loader.batch.deadlock.retry";
public static String DEADLOCK_WAIT_INTERVAL = "delta.loader.batch.deadlock.wait_interval";

public static String QUEUE_SIZE = "delta.loader.queue_size";
public static void init() {

try {
configation = new PropertiesConfiguration(
"smcmw_config.properties");
} catch (Exception e) {
log.error("Exception: ", e);
}
}

public static PropertiesConfiguration getConfiguration() {
return configation;
}

public static void main(String[] args) {
PropertyConfig.init();
System.out.println(PropertyConfig.getConfiguration().getString(PropertyConfig.DB_DRIVER));
System.out.println(PropertyConfig.getConfiguration().getString(PropertyConfig.DB_URL));
System.out.println(PropertyConfig.getConfiguration().getString(PropertyConfig.DEADLOCK_WAIT_INTERVAL));
}
}

>>>>BatchLoadTask.java
package com.citi.isg.smc.mw.delta;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;

import oracle.jdbc.OraclePreparedStatement;
import oracle.xdb.XMLType;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.citi.isg.smc.mw.delta.db.ConnectionPool;
import com.citi.isg.smc.mw.delta.db.DbmsOutput;
import com.citi.isg.smc.mw.delta.db.PooledConnection;
import com.citi.isg.smc.mw.delta.model.DeltaData;
import com.citi.isg.smc.mw.delta.model.FixedSizeList;

public class BatchLoadTask implements Callable<Integer> {
public static final String THREAD_NAME = "BatchLoadTask";
private static final AtomicInteger created = new AtomicInteger();
private static final AtomicInteger alive = new AtomicInteger();
private static Log log = LogFactory
.getLog(BatchLoadTask.class);
private FixedSizeList<DeltaData> deltaDataList;
private String name;
private int deadlockRetry;
private long deadlockWaitInterval;
private ConnectionPool connectionPool;

public BatchLoadTask(FixedSizeList<DeltaData> deltaDataList,
String name, ConnectionPool connectionPool, int deadlockRetry,
long deadlockWaitInterval) throws SQLException {
this.name = THREAD_NAME + "-" + name + "-" + created.incrementAndGet();
this.deltaDataList = deltaDataList;
this.connectionPool = connectionPool;
this.deadlockRetry = deadlockRetry;
this.deadlockWaitInterval = deadlockWaitInterval;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public Integer call() {
Thread curThread = Thread.currentThread();
log
.debug("Start Task " + getName() + " in Thread "
+ curThread.getId());

long starttime = System.currentTimeMillis();
PooledConnection pc = null;
Connection conn = null;
OraclePreparedStatement stmt = null;
int currentRetry = 0;
while (currentRetry <= deadlockRetry) {
try {
alive.incrementAndGet();
pc = connectionPool.getConnection();
conn = pc.getConnection();
conn.setAutoCommit(false);
stmt = (OraclePreparedStatement) conn
.prepareStatement("DECLARE smcp VARCHAR2(30); version NUMBER; mktsector VARCHAR2(10); modts TIMESTAMP(6); BEGIN DBMS_OUTPUT.ENABLE (buffer_size => NULL); smcp := ?; version := ?; mktsector := ?; modts := ?; INSERT INTO MW.SMCMW_DELTA (SMCP, MKT_SECTOR_CD, VERSION, DELTA_XML, MOD_TS) VALUES (smcp, mktsector, version, ?, modts); EXCEPTION WHEN DUP_VAL_ON_INDEX THEN DBMS_OUTPUT.PUT_LINE('ERROR: duplicate pk with smcp : '||smcp||', version : '||version||', market sector : '||mktsector||', mod_ts : '||modts); END;");

DbmsOutput dbmsOutput = new DbmsOutput(conn);
dbmsOutput.enable(32000);
for (DeltaData item : deltaDataList) {
XMLType xmlObj = XMLType.createXML(conn, item.getData());

stmt.setString(1, item.getSmcp());
stmt.setInt(2, item.getVersion());
stmt.setString(3, item.getType());
stmt.setTimestamp(4, new java.sql.Timestamp(item.getModTs()
.getTime()));
stmt.setObject(5, xmlObj);

stmt.addBatch();
}
stmt.executeBatch();
conn.commit();
stmt.clearBatch();

String outputStr = dbmsOutput.show();
if (outputStr != null && outputStr.length() > 0
&& !outputStr.equals("\n")) {
log.debug("PK violation: " + outputStr);
}
stmt.close();
return new Integer(deltaDataList.size());

} catch (SQLException exc) {
log.error("SQLException: ", exc);
try {
if (conn != null) {
conn.rollback();
}
} catch (Exception e) {
log.error("Exception: ", e);
}
if (exc.getMessage().indexOf("deadlock") >= 0) {
currentRetry++;
if (currentRetry > deadlockRetry) {
log.error("task " + getName()
+ " failed after all retries", exc);

} else {
if (currentRetry > 1) {
log.warn("task " + getName() + " " + currentRetry
+ " retries ", exc);
}
}
try {
if (stmt != null)
stmt.close();
Thread.sleep(deadlockWaitInterval);
connectionPool.resetConnection(pc);
} catch (Exception e) {
log.error("Exception: ", e);
}
} else {
log.error("task " + getName() + " failed", exc);

}
return 0;
} catch (Exception e) {
log.error("Exception: ", e);
return 0;
} finally {
alive.decrementAndGet();
long endtime = System.currentTimeMillis();
log.debug("End task " + getName() + " with data size "
+ deltaDataList.getDataSize() + " item size "
+ deltaDataList.size() + " in " + (endtime - starttime)
+ " millseconds with PooledConnection "+pc.getName());
connectionPool.releaseConnection(pc);
}
}
return 0;
}

public static int getThreadsCreated() {
return created.get();
}

public static int getThreadsAlive() {
return alive.get();
}
}

>>>>FullDeltaLoader.java
package com.citi.isg.smc.mw.delta;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.SQLException;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.citi.isg.smc.mw.delta.db.ConnectionPool;
import com.citi.isg.smc.mw.delta.model.DeltaData;
import com.citi.isg.smc.mw.delta.model.FixedSizeList;
import com.citi.isg.smc.mw.delta.model.ServiceQueue;
import com.citi.isg.smc.mw.delta.parser.QuickDeltaXMLParser;
import com.citi.isg.smc.mw.delta.utils.PropertyConfig;

public class FullDeltaLoader {
private static Log log = LogFactory
.getLog(FullDeltaLoader.class);

public static void main(String[] args) {
if (args.length == 0) {
log.debug("Need to specify filename");
return;
}
PropertyConfig.init();

String fileName = args[0];
FileInputStream inStream = null;
DataInputStream inFile = null;
BufferedReader inBr = null;
ExecutorService executor = null;
ConnectionPool connectionPool = null;
try {

long starttime = System.currentTimeMillis();

log.debug("Starting Binary XML Java Example with file name "
+ fileName);
connectionPool = new ConnectionPool(PropertyConfig
.getConfiguration().getInt(PropertyConfig.QUEUE_SIZE));

File deltaFile = new File(fileName);
inStream = new FileInputStream(deltaFile);
inFile = new DataInputStream(inStream);

inBr = new BufferedReader(new InputStreamReader(inFile));

String deltaStr = null;
executor = Executors.newFixedThreadPool(PropertyConfig
.getConfiguration().getInt(PropertyConfig.QUEUE_SIZE));
CompletionService<Integer> ecs = new ExecutorCompletionService<Integer>(
executor);
int batchCount = 0;
ServiceQueue<DeltaData> queue = new ServiceQueue<DeltaData>(
PropertyConfig.getConfiguration().getInt(
PropertyConfig.QUEUE_SIZE), PropertyConfig
.getConfiguration().getInt(
PropertyConfig.MAX_ITEM_SIZE),
PropertyConfig.getConfiguration().getInt(
PropertyConfig.MAX_DATA_SIZE));
int count = 0;
int maxLineSize = 0;
while ((deltaStr = inBr.readLine()) != null) {
if (deltaStr.length() > maxLineSize) {
maxLineSize = deltaStr.length();
}
QuickDeltaXMLParser parser = new QuickDeltaXMLParser(deltaStr);
if (parser != null) {
queue.add(new DeltaData(parser.getSmcp(), parser
.getVersion(), parser.getType(), deltaStr, parser
.getModTs()));
if (queue.batchPrepared()) {
ecs
.submit(new BatchLoadTask(
queue.getBatch(),
fileName,
connectionPool,
PropertyConfig
.getConfiguration()
.getInt(
PropertyConfig.DEADLOCK_RETRY),
PropertyConfig
.getConfiguration()
.getInt(
PropertyConfig.DEADLOCK_WAIT_INTERVAL)));
batchCount++;
}

}
count++;
}
FixedSizeList<DeltaData> batch = queue.getBatch();
if (batch != null && batch.size() > 0) {
ecs.submit(new BatchLoadTask(batch,
fileName, connectionPool, PropertyConfig
.getConfiguration().getInt(
PropertyConfig.DEADLOCK_RETRY),
PropertyConfig.getConfiguration().getInt(
PropertyConfig.DEADLOCK_WAIT_INTERVAL)));
batchCount++;
}

int rc = 0;
for (int i = 0; i < batchCount; i++) {
Integer size = ecs.take().get();
if (size != null) {
rc += size;
}
}
long endtime = System.currentTimeMillis();
log.debug("Completed Binary XML Java Example with file name "
+ fileName + " in " + (endtime - starttime)
+ " millis. File size: " + getFileSize(deltaFile)
+ " gigabytes. Total count : " + count + ". MaxLineSize : "
+ maxLineSize);

} catch (SQLException e) {
log.error("SQLException: ",e);
} catch (IOException e) {
log.error("IOException: ",e);
} catch (Exception e) {
log.error("Exception: ",e);
} finally {

if (executor != null)
executor.shutdownNow();

if (connectionPool != null)
connectionPool.close();

try {
if (inBr != null)
inBr.close();

if (inFile != null)
inFile.close();

if (inStream != null)
inStream.close();
} catch (IOException ioe) {
log.error("IO close exception : ", ioe);

}
}
}

public static double getFileSize(File file) {
double bytes = file.length();
double kilobytes = (bytes / 1024);
double megabytes = (kilobytes / 1024);
double gigabytes = (megabytes / 1024);

return gigabytes;
}
}

>>>>InterruptedBatchLoadTask.java
package com.citi.isg.smc.mw.delta;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;

import oracle.jdbc.OraclePreparedStatement;
import oracle.xdb.XMLType;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.citi.isg.smc.mw.delta.db.ConnectionPool;
import com.citi.isg.smc.mw.delta.db.PooledConnection;
import com.citi.isg.smc.mw.delta.model.DeltaData;
import com.citi.isg.smc.mw.delta.model.FixedSizeList;

public class InterruptedBatchLoadTask implements Callable<Integer> {
public static final String THREAD_NAME = "BatchLoadTask";
private static final AtomicInteger created = new AtomicInteger();
private static final AtomicInteger alive = new AtomicInteger();
private static Log log = LogFactory.getLog(InterruptedBatchLoadTask.class);
private FixedSizeList<DeltaData> deltaDataList;
private String name;
private ConnectionPool connectionPool;

public InterruptedBatchLoadTask(FixedSizeList<DeltaData> deltaDataList, String name,
ConnectionPool connectionPool) throws SQLException {
this.name = THREAD_NAME + "-" + name + "-" + created.incrementAndGet();
this.deltaDataList = deltaDataList;
this.connectionPool = connectionPool;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public Integer call() {
Thread curThread = Thread.currentThread();
log
.debug("Start Task " + getName() + " in Thread "
+ curThread.getId());

log.debug("Created " + getName());

long starttime = System.currentTimeMillis();
PooledConnection pc = null;
try {
alive.incrementAndGet();
pc = connectionPool.getConnection();
Connection conn = pc.getConnection();
conn.setAutoCommit(false);
OraclePreparedStatement stmt = (OraclePreparedStatement) conn
.prepareStatement("INSERT INTO SMCAPP.SMCMW_DELTA (SMCP, VERSION, MKT_SECTOR_CD, MOD_TS, DELTA_XML) VALUES (?, ?, ?, ?, ?)");

for (DeltaData item : deltaDataList) {
XMLType xmlObj = XMLType.createXML(conn, item.getData());

stmt.setString(1, item.getSmcp());
stmt.setInt(2, item.getVersion());
stmt.setString(3, item.getType());
stmt.setTimestamp(4, new java.sql.Timestamp(item.getModTs()
.getTime()));
stmt.setObject(5, xmlObj);

stmt.addBatch();
}
stmt.executeBatch();
conn.commit();
stmt.clearBatch();
stmt.close();

} catch (SQLException e) {
log.error("SQLException: ", e);
} finally {
alive.decrementAndGet();
long endtime = System.currentTimeMillis();
log.debug("End task " + getName() + " with " + deltaDataList.size()
+ " items in " + (endtime - starttime) + " millseconds with PooledConnection "+pc.getName());

connectionPool.releaseConnection(pc);
}
return new Integer(deltaDataList.size());
}

public static int getThreadsCreated() {
return created.get();
}

public static int getThreadsAlive() {
return alive.get();
}
}

>>>>>InterruptedFullDeltaLoader.java
package com.citi.isg.smc.mw.delta;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.SQLException;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.citi.isg.smc.mw.delta.db.ConnectionPool;
import com.citi.isg.smc.mw.delta.model.DeltaData;
import com.citi.isg.smc.mw.delta.model.FixedSizeList;
import com.citi.isg.smc.mw.delta.model.ServiceQueue;
import com.citi.isg.smc.mw.delta.parser.DeltaXMLParser;
import com.citi.isg.smc.mw.delta.utils.PropertyConfig;

public class InterruptedFullDeltaLoader {
private static Log log = LogFactory.getLog(InterruptedFullDeltaLoader.class);

public static void main(String[] args) {
if (args.length == 0) {
log.debug("Need to specify filename");
return;
}
PropertyConfig.init();
String fileName = args[0];
FileInputStream inStream = null;
DataInputStream inFile = null;
BufferedReader inBr = null;
ExecutorService executor = null;
ConnectionPool connectionPool = null;
try {

long starttime = System.currentTimeMillis();

log.debug("Starting Binary XML Java Example with file name "
+ fileName);

connectionPool = new ConnectionPool(PropertyConfig
.getConfiguration().getInt(PropertyConfig.QUEUE_SIZE));

File deltaFile = new File(fileName);
inStream = new FileInputStream(deltaFile);
inFile = new DataInputStream(inStream);

inBr = new BufferedReader(new InputStreamReader(inFile));

String deltaStr = null;
executor = Executors.newFixedThreadPool(PropertyConfig
.getConfiguration().getInt(PropertyConfig.QUEUE_SIZE));
CompletionService<Integer> ecs = new ExecutorCompletionService<Integer>(
executor);
int batchCount = 0;
ServiceQueue<DeltaData> queue = new ServiceQueue<DeltaData>(
PropertyConfig.getConfiguration().getInt(
PropertyConfig.QUEUE_SIZE), PropertyConfig
.getConfiguration().getInt(
PropertyConfig.MAX_ITEM_SIZE),
PropertyConfig.getConfiguration().getInt(
PropertyConfig.MAX_DATA_SIZE));
int count = 0;
while ((deltaStr = inBr.readLine()) != null) {
DeltaXMLParser parser = new DeltaXMLParser(deltaStr);
if (parser != null) {
queue.add(new DeltaData(parser.getSmcp(), parser
.getVersion(), parser.getType(), deltaStr, parser
.getModTs()));
if (queue.batchPrepared()) {
ecs
.submit(new InterruptedBatchLoadTask(queue.getBatch(),
fileName, connectionPool));
batchCount++;
}

}
count++;
}
FixedSizeList<DeltaData> batch = queue.getBatch();
if (batch != null && batch.size() > 0) {
ecs.submit(new InterruptedBatchLoadTask(batch, fileName, connectionPool));
batchCount++;
}

int rc = 0;
for (int i = 0; i < batchCount; i++) {
Integer size = ecs.take().get();
if (size != null) {
rc += size;
}
}
long endtime = System.currentTimeMillis();
log.debug("Completed Binary XML Java Example with file name "
+ fileName + " in " + (endtime - starttime)
+ " millis. File size: " + getFileSize(deltaFile)
+ " gigabytes. Total count : " + count + ".");

} catch (SQLException e) {
log.error("SQLException: ", e);
} catch (IOException e) {
log.error("IOException: ",e);
} catch (Exception e) {
log.error("Exception: ",e);
} finally {

if (executor != null)
executor.shutdownNow();

if (connectionPool != null)
connectionPool.close();

try {
if (inBr != null)
inBr.close();

if (inFile != null)
inFile.close();

if (inStream != null)
inStream.close();
} catch (IOException ioe) {
log.error("IO close exception : ", ioe);

}
}
}

public static double getFileSize(File file) {
double bytes = file.length();
double kilobytes = (bytes / 1024);
double megabytes = (kilobytes / 1024);
double gigabytes = (megabytes / 1024);

return gigabytes;
}

}

config file.
db.smc.driver=oracle.jdbc.driver.OracleDriver
db.smc.url=jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=smcdev2-scan.nam.nsroot.net) (PORT=15210))(CONNECT_DATA=(SERVICE_NAME=SMCD2_IEMW)))
db.smc.user=smcapi
db.smc.password=smcapi

db.smc.connectionpool.size=10

delta.loader.batch.max_item=2000
delta.loader.batch.max_total_data_size=5242880

delta.loader.batch.deadlock.retry=5
delta.loader.batch.deadlock.wait_interval=15000

delta.loader.queue_size=5

数据库连接线程池 批处理

猜你喜欢

数据库连接线程池批处理