cdh solr自定义索引, 导入mysql数据

solrctl instancedir --generate /var/lib/solr/config/test

solrctl instancedir --create collection_jdbc /var/lib/solr/config/test
solrctl instancedir --list

solrctl collection --create test # -s 1 -r 1 -m 1 
solrctl collection --list

solr 配置概览: solrchonfig.xml, schema.xml

1. 创建collection配置

#1.创建集合-配置文件
 #instancedir [--generate path [-schemaless]]
 #               [--create name path]
 #               [--update name path]
 #               [--get name path]
 #               [--delete name]
 #               [--list]

solrctl instancedir --generate /var/lib/solr/collection_jdbc
			      			 #/var/lib/solr/collection_jdbc/conf这个目录下产生很多配置文件

2. 创建/修改配置文件,并上传到 zookeeper

a, solrconfig.xml

• 1, solrconfig.xml 添加data-import相关jar包
  
• 2, 配置solrconfig.xml中requestHandle引用到的配置文件db-data-config.xml
  

b, schema.xml

solr 字段定义说明：https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html

• 1, schema.xml 添加自定义索引字段
  
• 2, schema.xml 删除默认id字段，修改uniqueKey使用自定义的字段名
  
  

#2.创建配置文件,并上传到 zookeeper
#添加配置: 使用data-importor,来抽取jdbc数据
sed -i '/luceneMatchVersion/a <lib dir="/var/lib/solr/jars/" regex=".*\\.jar" /> ' /var/lib/solr/collection_jdbc/conf/solrconfig.xml 
sed -i '/<\/requestDispatcher/a \
  <requestHandler name="/dataimport" class="solr.DataImportHandler"> \
    <lst name="defaults"> \
      <str name="config">db-data-config.xml</str> \
    </lst> \
  </requestHandler> ' \
/var/lib/solr/collection_jdbc/conf/solrconfig.xml 

#配置data-import
[root@test-c6 ~]# cat /var/lib/solr/collection_jdbc/conf/db-data-config.xml
<dataConfig>
    <dataSource driver="com.mysql.jdbc.Driver" url="jdbc:mysql://192.168.56.161:3306/test" user="root" password="123456" />
    <document>
        <entity name="t1" query="select * from t1"
                deltaQuery="select * from t1  where last_modified > '${dataimporter.last_index_time}'">
            <field column="name" name="my_name" />
            <field column="id" name="my_id" />
            <field column="age" name="my_age" />
        </entity>
    </document>
</dataConfig>


#拷贝jar包到指定的位置
[root@eadage img]# ls /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler*
/opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler-4.10.3-cdh5.12.0.jar         /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler-extras.jar
/opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler-extras-4.10.3-cdh5.12.0.jar  /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler.jar

[root@test-c6 ~]# ls /var/lib/solr/jars/ 
mysql-connector-java.jar                     solr-dataimporthandler-extras-4.10.3-cdh5.12.0.jar  solr-dataimporthandler.jar
solr-dataimporthandler-4.10.3-cdh5.12.0.jar  solr-dataimporthandler-extras.jar

#添加配置: schema.xml, 字段映射
sed -i '/<fields>/a <field name="my_id" type="string"	      required="true" indexed="true" stored="true"  multiValued="false" /> ' /var/lib/solr/collection_jdbc/conf/schema.xml
sed -i '/<fields>/a <field name="my_name" type="text_general"		      indexed="true" stored="true"  multiValued="false" /> ' /var/lib/solr/collection_jdbc/conf/schema.xml
sed -i '/<fields>/a <field name="my_age" type="int"			      indexed="true" stored="true"  multiValued="false" /> ' /var/lib/solr/collection_jdbc/conf/schema.xml
#替换旧的默认id, 指定uniquekey为自己的字段名
sed -i 's@<uniqueKey>id</uniqueKey>@<uniqueKey>my_id</uniqueKey>@' /var/lib/solr/collection_jdbc/conf/schema.xml
sed -i '/<field name="id"/d'  /var/lib/solr/collection_jdbc/conf/schema.xml 

#上传配置到zk
solrctl instancedir --create collection_jdbc /var/lib/solr/collection_jdbc
solrctl instancedir --list

3. 创建 collection

#3.创建 collection:
#collection  [--create name -s <numShards>
#                             [-a Create collection with autoAddReplicas=true]
#                             [-c <collection.configName>]
#                             [-r <replicationFactor>]
#                             [-m <maxShardsPerNode>]
#                             [-n <createNodeSet>]]
#               [--delete name]
#               [--reload name]
#               [--stat name]
#               [--deletedocs name]
#               [--list]

solrctl collection --create collection_jdbc # -s 1 -r 1 -m 1 
solrctl collection --list

4. 使用solr admin ：导入mysql数据

• 登录mysql: 查看数据

mysql> select * from  test.t1;
+------+------+------+
| id   | name | age  |
+------+------+------+
| NULL | a    |   28 |
| 2    | b    |   24 |
| 3    | c    |   22 |
+------+------+------+
3 rows in set (0.00 sec)

• 使用solr admin导入数据
  
• 查询数据
  

5. 修改solr.xml, solrconfig.xml，添加字段

a, solrconfig.xml

• solrsconfig.xml中引用的db-data-config.xml添加字段
  

b, schema.xml

• schema.xml添加一个字段
  
• 重载配置

[root@test-c6 ~]# solrctl instancedir --update collection_jdbc /var/lib/solr/collection_jdbc/
Uploading configs from /var/lib/solr/collection_jdbc//conf to test-c6:2181/solr. This may take up to a minute.

[root@test-c6 ~]# solrctl collection --reload collection_jdbc
[root@test-c6 ~]#

• 再次导入数据，并查询

mysql> select * from  test.t1;
+------+------+------+---------------------+
| id   | name | age  | regist_time         |
+------+------+------+---------------------+
| 1    | a    |   28 | 2018-10-10 10:12:10 |
| 2    | b    |   24 | 2019-09-00 09:00:00 |
| 3    | c    |   22 | 2018-10-10 10:12:10 |
| 4    | NULL | NULL | 2018-10-10 10:12:10 |
| 5    | e    |   33 | 2020-11-27 10:27:00 |
+------+------+------+---------------------+
5 rows in set (0.00 sec)