kettle7.0写入elasticsearch5.11插件

kettle7.0写入elasticsearch5.11插件

kettle7.0中自带es版本支持为2.3，但是5.x版本获取客户端连接的api已改，因此需要改源码重新编译。

下载kettle源码后，只提取es部分

建立普通java工程，目录层级与上面一致，导入依赖jar包。

jar包参照安装好的pentaho-kettle-7.0.0.0-R\plugins\elasticsearch-bulk-insert\lib下的从安装好的es5.11中提取对应版本的jar包。

更改 src\org\pentaho\di\trans\steps\elasticsearchbulk\ElasticSearchBulk.java

//import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.transport.client.PreBuiltTransportClient; 

private void initClient() throws UnknownHostException {

    Settings.Builder settingsBuilder = Settings.builder();
    settingsBuilder.put( Settings.Builder.EMPTY_SETTINGS ); // keep default classloader
    settingsBuilder.put( meta.getSettingsMap() );
    Settings settings = settingsBuilder.build();
    client = new PreBuiltTransportClient(settings);
//    //TransportClient.Builder tClientBuilder = TransportClient.builder().settings( settingsBuilder );
//    TransportClient client = new PreBuiltTransportClient(settings);  
//    
    if ( !meta.servers.isEmpty() ) {
      node = null;
      //TransportClient tClient = tClientBuilder.build();
      for ( ElasticSearchBulkMeta.Server s : meta.servers ) {
          ((TransportClient) client).addTransportAddress(s.getAddr());
      }
      //client = tClient;
    }
//    } else {
//      NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder();
//      nodeBuilder.settings( settingsBuilder );
//      node = nodeBuilder.client( true ).node(); // this node will not hold data
//      client = node.client();
//      node.start();
//    }

//      for ( ElasticSearchBulkMeta.Server s : meta.servers ) {
//        logError(s.getAddr().toString());
//      }
//      
//      logError("-----");
//      Settings settings = Settings.builder().put("cluster.name", "estest").build();;// 集群名
//    client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("10.0.0.20"), 12300));  
  }

  private void disposeClient() {

    if ( client != null ) {
      client.close();
    }
//    if ( node != null ) {
//      node.close();
//    }

  }

更改 src\org\pentaho\di\ui\trans\steps\elasticsearchbulk\ElasticSearchBulkDialog.java

  private void test( TestType testType ) {

    // Save off the thread's context class loader to restore after the test
    ClassLoader originalClassloader = Thread.currentThread().getContextClassLoader();

    // Now ensure that the thread's context class loader is the plugin's classloader
    Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader() );

    TransportClient client = null;
    Node node = null;
    try {

      ElasticSearchBulkMeta tempMeta = new ElasticSearchBulkMeta();
      toModel( tempMeta );

      Settings.Builder settingsBuilder = Settings.builder();
      settingsBuilder.put( Settings.Builder.EMPTY_SETTINGS ); // keep default classloader
      settingsBuilder.put( tempMeta.getSettingsMap() );
      Settings settings = settingsBuilder.build();
      client = new PreBuiltTransportClient(settings);

      if ( !tempMeta.getServers().isEmpty() ) {
        node = null;
        //TransportClient tClient = tClientBuilder.build();
        for ( ElasticSearchBulkMeta.Server s : tempMeta.getServers() ) {
          ((TransportClient) client).addTransportAddress(s.getAddr());
        }
        //client = tClient;
      }
////      TransportClient.Builder tClientBuilder = TransportClient.builder().settings( settingsBuilder );
//      Settings settings = settingsBuilder.build();
//       client = new PreBuiltTransportClient(settings);  
//      if ( !tempMeta.getServers().isEmpty() ) {
//        node = null;
////        TransportClient tClient = tClientBuilder.build();
//        for ( ElasticSearchBulkMeta.Server s : tempMeta.getServers() ) {
//          client.addTransportAddress( s.getAddr() );
//        }
////        client = tClient;
//      } 
//      else {
//        NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder();
//        nodeBuilder.settings( settingsBuilder );
//        node = nodeBuilder.client( true ).node();
//        client = node.client();
//        node.start();
//      }
//      logError(settingsBuilder.toString());
//      logError("-----");
//      Settings settings = Settings.builder().put("cluster.name", "estest").build();;// 集群名
//    client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("10.0.0.20"), 12300));
      AdminClient admin = client.admin();

      switch ( testType ) {
        case INDEX:
          if ( StringUtils.isBlank( tempMeta.getIndex() ) ) {
            showError( BaseMessages.getString( PKG, "ElasticSearchBulk.Error.NoIndex" ) );
            break;
          }
          // First check to see if the index exists
          IndicesExistsRequestBuilder indicesExistBld = admin.indices().prepareExists( tempMeta.getIndex() );
          IndicesExistsResponse indicesExistResponse = indicesExistBld.execute().get();
          if ( !indicesExistResponse.isExists() ) {
            showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoIndex" ) );
            return;
          }

          RecoveryRequestBuilder indicesBld = admin.indices().prepareRecoveries( tempMeta.getIndex() );
          ListenableActionFuture<RecoveryResponse> lafInd = indicesBld.execute();
          String shards = "" + lafInd.get().getSuccessfulShards() + "/" + lafInd.get().getTotalShards();
          showMessage( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.TestIndex.TestOK", shards ) );
          break;
        case CLUSTER:
          ClusterStateRequestBuilder clusterBld = admin.cluster().prepareState();
          ListenableActionFuture<ClusterStateResponse> lafClu = clusterBld.execute();
          ClusterStateResponse cluResp = lafClu.actionGet();
          String name = cluResp.getClusterName().value();
          ClusterState cluState = cluResp.getState();
          int numNodes = cluState.getNodes().getSize();
          showMessage( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.TestCluster.TestOK", name, numNodes ) );
          break;
        default:
          break;
      }
    } catch ( NoNodeAvailableException e ) {
      showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoNodesFound" ) );
    } catch ( MasterNotDiscoveredException e ) {
      showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoNodesFound" ) );
    } catch ( Exception e ) {
      showError( e.getLocalizedMessage() );
    } finally {
      if ( client != null ) {
        client.close();
      }
//      if ( node != null ) {
//        node.close();
//      }
    }

    // Restore the original classloader
    Thread.currentThread().setContextClassLoader( originalClassloader );

  }

导出为elasticsearch-bulk-insert-plugin-7.0.0.0-25.jar，将jar及lib替换原来的即可。

插件lib替换注意，在原2.3插件的基础上，复制es6安装目录下lib及modules下所有jar至插件lib下，然后删除相同名称版本号较小的jar。

源码及打包好的插件见网盘：链接: https://pan.baidu.com/s/1F5-6mGiV_LgFf912-hc6rw 密码: cun9