init
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
package org.bdware.datanet.searchengine;
|
||||
|
||||
import com.google.gson.JsonObject;
|
||||
|
||||
//RepoMeta要求是一个jsonObject,要求必须要有一个doi
|
||||
public class DoMeta {
|
||||
JsonObject metadata;
|
||||
DoMeta(JsonObject jo){
|
||||
metadata=jo.deepCopy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return metadata.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package org.bdware.datanet.searchengine;
|
||||
|
||||
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import org.bdware.sc.engine.JSONTool;
|
||||
import org.junit.Test;
|
||||
import org.neo4j.graphdb.*;
|
||||
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
|
||||
import org.neo4j.graphdb.index.Index;
|
||||
import org.neo4j.graphdb.traversal.Evaluators;
|
||||
import org.neo4j.graphdb.traversal.TraversalDescription;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
public class MetaGraph {
|
||||
|
||||
// private static final String DB_PATH = "testgraph.db";
|
||||
|
||||
private static String DB_PATH="./MetaGraph.db";
|
||||
// private static enum LABEL {
|
||||
// DO, REAL, STANDARD
|
||||
// }
|
||||
|
||||
private GraphDatabaseService graphDB;
|
||||
public MetaGraph(String name){
|
||||
DB_PATH="./"+name+".db";
|
||||
graphDB = new GraphDatabaseFactory().newEmbeddedDatabase(new File(DB_PATH));
|
||||
}
|
||||
//DO类型
|
||||
|
||||
|
||||
/*
|
||||
{
|
||||
"doid":"xxx",
|
||||
"maskinfo":{
|
||||
"stu_no":"@int(5,10)",
|
||||
"name":"cname"
|
||||
}
|
||||
}
|
||||
*/
|
||||
public void createDoNode(String doid,JsonObject jo) {
|
||||
Transaction tx = graphDB.beginTx();
|
||||
Node Do = graphDB.createNode(new NodeLabel("DO"));//命名规则
|
||||
Do.setProperty("doid",doid);
|
||||
Iterator iter = jo.entrySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
Map.Entry entry = (Map.Entry) iter.next();
|
||||
Do.setProperty(entry.getKey().toString(),entry.getValue().toString());
|
||||
}
|
||||
tx.success();
|
||||
tx.close();
|
||||
//配置do的mask
|
||||
}
|
||||
public void iterDO() {
|
||||
|
||||
Transaction tx = graphDB.beginTx();
|
||||
ResourceIterator<Node> iterator = graphDB.findNodes(new NodeLabel("DO"));
|
||||
for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
|
||||
Node node = it.next();
|
||||
String doid=(String)node.getProperty("doid");
|
||||
String id=(String)node.getProperty("id");
|
||||
System.out.println(doid + " " + id+ " ");
|
||||
}
|
||||
|
||||
tx.success();
|
||||
tx.close();
|
||||
}
|
||||
//param:{"doid1":"xxx","doid2":"yyy","key1":"studentid","key2":"studentid"}
|
||||
public void createRelationDo2Do(JsonObject jo){
|
||||
Transaction tx = graphDB.beginTx();
|
||||
String doid1=jo.get("id1").getAsString();
|
||||
String doid2=jo.get("id2").getAsString();
|
||||
String key1=jo.get("key1").getAsString();
|
||||
String key2=jo.get("key2").getAsString();
|
||||
|
||||
Node node1=graphDB.findNode(new NodeLabel("DO"),"doid",doid1);
|
||||
Node node2=graphDB.findNode(new NodeLabel("DO"),"doid",doid2);
|
||||
System.out.println(jo);
|
||||
|
||||
node1.createRelationshipTo(node2, new RelationType(jo.toString()));
|
||||
tx.success();
|
||||
tx.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
//真实标签类型
|
||||
public void createRealNode(String key,String value) {
|
||||
Transaction tx = graphDB.beginTx();
|
||||
Node realField = graphDB.createNode(new NodeLabel("realField"));//命名规则
|
||||
realField.setProperty("key",key);
|
||||
realField.setProperty("value",value);
|
||||
tx.success();
|
||||
tx.close();
|
||||
|
||||
}
|
||||
|
||||
//标准标签
|
||||
|
||||
public void createStandardNode(String key,String value) {
|
||||
Transaction tx = graphDB.beginTx();
|
||||
Node standardField = graphDB.createNode(new NodeLabel("standardField"));//命名规则
|
||||
standardField.setProperty("key",key);
|
||||
standardField.setProperty("value",value);
|
||||
tx.success();
|
||||
tx.close();
|
||||
}
|
||||
public void iterRealField() {
|
||||
|
||||
Transaction tx = graphDB.beginTx();
|
||||
ResourceIterator<Node> iterator = graphDB.findNodes(new NodeLabel("DO"));
|
||||
for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
|
||||
Node node = it.next();
|
||||
String doid=(String)node.getProperty("doid");
|
||||
String id=(String)node.getProperty("id");
|
||||
System.out.println(doid + " " + id+ " ");
|
||||
}
|
||||
|
||||
tx.success();
|
||||
tx.close();
|
||||
}
|
||||
|
||||
public void iterStandardField() {
|
||||
|
||||
Transaction tx = graphDB.beginTx();
|
||||
ResourceIterator<Node> iterator = graphDB.findNodes(new NodeLabel("DO"));
|
||||
for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
|
||||
Node node = it.next();
|
||||
String doid=(String)node.getProperty("doid");
|
||||
String id=(String)node.getProperty("id");
|
||||
System.out.println(doid + " " + id+ " ");
|
||||
}
|
||||
|
||||
tx.success();
|
||||
tx.close();
|
||||
}
|
||||
// public void init() {
|
||||
// graphDB = new GraphDatabaseFactory().newEmbeddedDatabase(new File(DB_PATH));
|
||||
// }
|
||||
|
||||
// /**创建数据*/
|
||||
// public void create() {
|
||||
// Transaction tx = graphDB.beginTx();
|
||||
//
|
||||
// Node student = graphDB.createNode(new CaseLabel("STUDENT"));//命名规则
|
||||
// student.setProperty("name", "@student");//数据类型命名规则
|
||||
// student.setProperty("id", "@int(2001210000,2001210999)");
|
||||
//
|
||||
// Node course = graphDB.createNode(new CaseLabel("COURSE"));//命名规则
|
||||
// course.setProperty("name", "[枚举or读文件]");//数据类型命名规则
|
||||
// course.setProperty("id", "@int(10000,10999)");
|
||||
//
|
||||
// student.createRelationshipTo(course, new RelTypes("CRIME"));
|
||||
//
|
||||
//
|
||||
//
|
||||
// Node case1 = graphDB.createNode(new CaseLabel("CASEINFO"));
|
||||
// case1.setProperty("name", "案件1");
|
||||
// case1.setProperty("address", "南山");
|
||||
//
|
||||
// Node case2 = graphDB.createNode(new CaseLabel("CASEINFO"));
|
||||
// case2.setProperty("name", "案件2");
|
||||
// case2.setProperty("address", "福田");
|
||||
//
|
||||
// Node case3 = graphDB.createNode(new CaseLabel("CASEINFO"));
|
||||
// case3.setProperty("name", "案件3");
|
||||
// case3.setProperty("address", "龙华");
|
||||
//
|
||||
// Node userA = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userA.setProperty("name", "A");
|
||||
// userA.setProperty("idcard", "150302198012228239");
|
||||
// userA.setProperty("tel", "13685246639");
|
||||
//
|
||||
// Node userB = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userB.setProperty("name", "B");
|
||||
// userB.setProperty("idcard", "370634199208304929");
|
||||
// userB.setProperty("tel", "13885246670");
|
||||
//
|
||||
// Node userC = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userC.setProperty("name", "C");
|
||||
// userC.setProperty("idcard", "430721198802065735");
|
||||
// userC.setProperty("tel", "13966704782");
|
||||
//
|
||||
// Node userD = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userD.setProperty("name", "D");
|
||||
// userD.setProperty("idcard", "522730198707118747");
|
||||
// userD.setProperty("tel", "13670478962");
|
||||
//
|
||||
// Node userE = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userE.setProperty("name", "E");
|
||||
// userE.setProperty("idcard", "622926198609158032");
|
||||
// userE.setProperty("tel", "13047829667");
|
||||
//
|
||||
// Node userF = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userF.setProperty("name", "F");
|
||||
// userF.setProperty("idcard", "500114197706138305");
|
||||
// userF.setProperty("tel", "13478296670");
|
||||
//
|
||||
// Node userG = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userG.setProperty("name", "G");
|
||||
// userG.setProperty("idcard", "500114106138305152");
|
||||
// userG.setProperty("tel", "13476670156");
|
||||
//
|
||||
// Node userH = graphDB.createNode(new CaseLabel("PERSON"));
|
||||
// userH.setProperty("name", "H");
|
||||
// userH.setProperty("idcard", "500114197704751236");
|
||||
// userH.setProperty("tel", "13296156670");
|
||||
//
|
||||
// case1.createRelationshipTo(userA, new RelTypes("CRIME"));
|
||||
// case1.createRelationshipTo(userB, new RelTypes("CRIME"));
|
||||
// case1.createRelationshipTo(userC, new RelTypes("CRIME"));
|
||||
// case1.createRelationshipTo(userD, new RelTypes("CRIME"));
|
||||
//
|
||||
// case2.createRelationshipTo(userB, new RelTypes("CRIME"));
|
||||
// case2.createRelationshipTo(userC, new RelTypes("CRIME"));
|
||||
// case2.createRelationshipTo(userE, new RelTypes("CRIME"));
|
||||
//// case2.createRelationshipTo(userD, RelTypes.CRIME);
|
||||
//
|
||||
//
|
||||
// case3.createRelationshipTo(userF, new RelTypes("CRIME"));
|
||||
// case3.createRelationshipTo(userG, new RelTypes("CRIME"));
|
||||
// case3.createRelationshipTo(userH, new RelTypes("CRIME"));
|
||||
//
|
||||
// tx.success();
|
||||
// tx.close();
|
||||
//
|
||||
// }
|
||||
//
|
||||
// public void search(String username) {
|
||||
// Transaction tx = graphDB.beginTx();
|
||||
// System.out.println(graphDB.getAllNodes().stream().findFirst());
|
||||
// Node startNode = graphDB.findNode( "name", username);
|
||||
//
|
||||
// Iterable<Relationship> iterable = startNode
|
||||
// .getRelationships(new RelTypes("CRIME"), Direction.INCOMING);
|
||||
// for (Relationship r : iterable) {
|
||||
//
|
||||
// Node node = r.getStartNode();
|
||||
// long id = node.getId();
|
||||
// String name = (String)node.getProperty("name");
|
||||
//// String idcard = (String)node.getProperty("idcard");
|
||||
//// String tel = (String)node.getProperty("tel");
|
||||
//
|
||||
// System.out.println(id + " " + name+ " ");
|
||||
// }
|
||||
//
|
||||
// tx.success();
|
||||
// tx.close();
|
||||
// }
|
||||
//
|
||||
// /**遍历查找**/
|
||||
// public void searchUser(String username) {
|
||||
// Transaction tx = graphDB.beginTx();
|
||||
//
|
||||
// Node startNode = graphDB.findNode(new CaseLabel("PERSON"), "name", username);
|
||||
//// Node startNode = graphDB.findNode(new CaseLabel("PERSON"), "name", "B");
|
||||
//
|
||||
// TraversalDescription td = graphDB.traversalDescription();
|
||||
// td.relationships(new RelTypes("CRIME"), Direction.OUTGOING);//沿着关系 TONG
|
||||
// td.depthFirst();//设置深度优先
|
||||
//// td.evaluator(Evaluators.excludeStartPosition());
|
||||
//
|
||||
// //从开始节点开始
|
||||
//
|
||||
// Iterable<Node> it = td.traverse(startNode).nodes();
|
||||
// for (Node node : it) {
|
||||
// long id = node.getId();
|
||||
// if( node.hasLabel( new CaseLabel("PERSON") )) {
|
||||
// String name = (String)node.getProperty("name");
|
||||
// String idcard = (String)node.getProperty("idcard");
|
||||
// String tel = (String)node.getProperty("tel");
|
||||
//
|
||||
// System.out.println(id + " " + name+ " " + idcard + " " + tel);
|
||||
// } else {
|
||||
//// String name = (String)node.getProperty("name");
|
||||
//// String address = (String)node.getProperty("address");
|
||||
//// System.out.println(id + " " + name+ " " + address);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// tx.success();
|
||||
// tx.close();
|
||||
// }
|
||||
//
|
||||
|
||||
|
||||
//
|
||||
// /**查询所有作案人员信息*/
|
||||
// public void searchAllPerson() {
|
||||
//
|
||||
//
|
||||
//
|
||||
// Transaction tx = graphDB.beginTx();
|
||||
//
|
||||
// ResourceIterator<Node> iterator = graphDB.findNodes(new CaseLabel("PERSON"));
|
||||
// for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
|
||||
// Node node = it.next();
|
||||
// long id = node.getId();
|
||||
// String name = (String)node.getProperty("name");
|
||||
// String idcard = (String)node.getProperty("idcard");
|
||||
// String tel = (String)node.getProperty("tel");
|
||||
//
|
||||
// System.out.println(id + " " + name+ " " + idcard + " " + tel);
|
||||
// }
|
||||
//
|
||||
// tx.success();
|
||||
// tx.close();
|
||||
// }
|
||||
}
|
||||
|
||||
class RelationType implements RelationshipType {
|
||||
private String name;
|
||||
public RelationType(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String name(){
|
||||
return name;
|
||||
}
|
||||
}
|
||||
class NodeLabel implements Label {
|
||||
private String name;
|
||||
public NodeLabel(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String name(){
|
||||
return name;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
package org.bdware.datanet.searchengine;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||
import org.apache.lucene.document.*;
|
||||
import org.apache.lucene.index.*;
|
||||
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.bdware.sc.boundry.utils.RocksDBUtil;
|
||||
import org.rocksdb.RocksIterator;
|
||||
import org.wltea.analyzer.lucene.IKAnalyzer;
|
||||
import java.io.*;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
public class MetaIndex {
|
||||
private final Logger LOGGER = LogManager.getLogger(MetaIndex.class);
|
||||
public FSDirectory indexDir;
|
||||
public boolean isEmpty = false;//描述当前repo的文件是否存在
|
||||
private IndexWriter indexWriter;
|
||||
private static final int COUNT=20;
|
||||
private static final int OFFSET=0;
|
||||
MetaIndex(String name){
|
||||
initIndex(name);
|
||||
}
|
||||
MetaIndex(){
|
||||
initIndex("SearchEngineIndex"+System.currentTimeMillis());
|
||||
}
|
||||
|
||||
public void initIndex(String Dirname) {
|
||||
try {
|
||||
File dir = new File(Dirname);
|
||||
if (!dir.exists()) {
|
||||
LOGGER.info("make metaIndex dir ");
|
||||
dir.mkdirs();
|
||||
isEmpty = true;
|
||||
}
|
||||
indexDir = FSDirectory.open(Paths.get(dir.toURI()));
|
||||
Analyzer analyzer = new IKAnalyzer();
|
||||
indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(analyzer));
|
||||
indexWriter.commit();
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
public void indexRepo(DoMeta meta) throws IOException {
|
||||
try {
|
||||
//IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig());
|
||||
Document document = new Document();
|
||||
// System.out.println(meta.metadata);
|
||||
for (Map.Entry entry : meta.metadata.entrySet()) {
|
||||
String key=entry.getKey().toString();
|
||||
System.out.println(entry.getValue());
|
||||
String val=entry.getValue().toString().substring(1,entry.getValue().toString().length()-1);
|
||||
System.out.println(key+" "+val);
|
||||
//if(key.equals("doid")||key.equals("owner"))
|
||||
// document.add(new StringField(key,val, Field.Store.YES));
|
||||
//范围类型
|
||||
if(key.equals("createTime")||key.equals("createDate")){
|
||||
document.add(new StringField(key,val,Field.Store.YES));
|
||||
}
|
||||
|
||||
else if(key.equals("doId")){
|
||||
document.add(new StringField(key,val,Field.Store.YES));
|
||||
//document.add(new LongPoint(key,Long.valueOf(val)),Field.Store.YES);
|
||||
}
|
||||
|
||||
else{
|
||||
document.add(new TextField(key,val, Field.Store.YES));
|
||||
}
|
||||
|
||||
//System.out.println("eee"+entry.getKey().toString()+" "+entry.getValue().toString().substring(1,entry.getValue().toString().length()-1));
|
||||
}
|
||||
indexWriter.addDocument(document);
|
||||
indexWriter.commit();
|
||||
isEmpty = false;
|
||||
}
|
||||
catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void updateRepoList(List<DoMeta> DoRepos)
|
||||
throws IOException, ParseException {
|
||||
IndexReader indexReader = null;
|
||||
for (DoMeta r : DoRepos) {
|
||||
String doId=r.metadata.get("doId").toString();
|
||||
if(isEmpty){
|
||||
indexRepo(r);
|
||||
LOGGER.info("--> index when dir is not empty");
|
||||
LOGGER.info("contract " + doId + " --> actually to index");
|
||||
}
|
||||
else{
|
||||
try {
|
||||
indexReader = DirectoryReader.open(indexDir);
|
||||
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
|
||||
LOGGER.debug( doId+ "--> try to index");
|
||||
//Query query = new TermQuery(new Term("doid", "AIBD/doip.Repo1001"));
|
||||
Query query = new TermQuery(new Term("doId", doId));
|
||||
TopDocs docs = indexSearcher.search(query, 10);
|
||||
|
||||
|
||||
LOGGER.debug(docs.scoreDocs);
|
||||
|
||||
if(docs.scoreDocs == null || docs.scoreDocs.length == 0) {
|
||||
indexRepo(r);
|
||||
LOGGER.info("--> index when dir is not empty");
|
||||
LOGGER.info("contract " + doId + " --> actually to index");
|
||||
}
|
||||
docs = indexSearcher.search(query, 10);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("getting index failed! " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if (null != indexReader) {
|
||||
indexReader.close();
|
||||
}
|
||||
}
|
||||
public void updateRepo(DoMeta DoRepo)
|
||||
throws IOException, ParseException {
|
||||
IndexReader indexReader = null;
|
||||
|
||||
String doId=DoRepo.metadata.get("doId").getAsString();
|
||||
if(isEmpty){
|
||||
indexRepo(DoRepo);
|
||||
LOGGER.info("--> index when dir is not empty");
|
||||
LOGGER.info("contract " + doId + " --> actually to index");
|
||||
}
|
||||
else{
|
||||
try {
|
||||
indexReader = DirectoryReader.open(indexDir);
|
||||
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
|
||||
LOGGER.debug( doId+ "--> try to index");
|
||||
Query query = new TermQuery(new Term("doId", doId));
|
||||
TopDocs docs = indexSearcher.search(query, 10);
|
||||
LOGGER.debug(docs.scoreDocs);
|
||||
System.out.println("bb"+docs.scoreDocs.length);
|
||||
if(docs.scoreDocs == null || docs.scoreDocs.length == 0) {
|
||||
indexRepo(DoRepo);
|
||||
LOGGER.info("--> index when dir is not empty");
|
||||
LOGGER.info("--> index when dir is not empty");
|
||||
LOGGER.info("contract " + doId + " --> actually to index");
|
||||
}
|
||||
docs = indexSearcher.search(query, 10);
|
||||
} catch (Exception e) {
|
||||
LOGGER.warn("getting index failed! " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (null != indexReader) {
|
||||
indexReader.close();
|
||||
}
|
||||
}
|
||||
public void deleteRepo(String doId) {
|
||||
|
||||
try {
|
||||
IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig());
|
||||
Query query=new TermQuery( new Term("doId",doId));
|
||||
System.out.println("query"+query.toString());
|
||||
indexWriter.deleteDocuments(query);
|
||||
indexWriter.commit();
|
||||
indexWriter.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
//模糊搜索
|
||||
public String search(JsonObject jo) {
|
||||
try {
|
||||
if (!jo.has("type")) {
|
||||
return "missing arguments";
|
||||
}
|
||||
String type=jo.get("type").getAsString();
|
||||
if (!jo.has("keyword")) return "missing arguments: keyword";
|
||||
int offset=OFFSET;
|
||||
if (jo.has("offset")) offset= jo.get("offset").getAsInt();
|
||||
offset= Math.max(offset, 0);
|
||||
int count=COUNT;
|
||||
if (jo.has("count")) count= jo.get("count").getAsInt();
|
||||
count=count<0?COUNT:count;
|
||||
String keyword = jo.get("keyword").getAsString();
|
||||
System.out.println(type+" "+keyword);
|
||||
// Analyzer analyzer = new IKAnalyzer();
|
||||
// QueryParser queryParser = new QueryParser(type, analyzer);
|
||||
// Query rmQuery = queryParser.parse(keyword);
|
||||
|
||||
DirectoryReader indexReader = DirectoryReader.open(indexDir);
|
||||
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
|
||||
Query query = new TermQuery(new Term(type, keyword));
|
||||
TopDocs docs = indexSearcher.search(query, 10);
|
||||
System.out.println("length "+docs.scoreDocs.length);
|
||||
ResultModel resultModel = null;
|
||||
if (docs.scoreDocs != null && docs.scoreDocs.length > 0)
|
||||
resultModel = paginate(docs, indexReader,count,offset);
|
||||
|
||||
if (resultModel != null) {
|
||||
return resultModel.toString();
|
||||
}
|
||||
ResultModel noDataResult=new ResultModel();
|
||||
return noDataResult.toString();
|
||||
}
|
||||
catch (Exception e) {
|
||||
ByteArrayOutputStream bo = new ByteArrayOutputStream();
|
||||
e.printStackTrace(new PrintStream(bo));
|
||||
return bo.toString();
|
||||
}
|
||||
}
|
||||
//按照范围查询
|
||||
public String searchByRange(JsonObject jo) {
|
||||
try {
|
||||
if (!jo.has("type")) {
|
||||
return "missing arguments";
|
||||
}
|
||||
//if (!jo.has("keyword")) return "missing arguments: keyword";
|
||||
if (!jo.has("from")) return "missing arguments: from";
|
||||
if (!jo.has("to")) return "missing arguments: to";
|
||||
String type = jo.get("type").getAsString();
|
||||
//String keyword = jo.get("keyword").getAsString();
|
||||
String from = jo.get("from").getAsString();
|
||||
String to = jo.get("to").getAsString();
|
||||
|
||||
int offset = OFFSET;
|
||||
if (jo.has("offset")) offset = jo.get("offset").getAsInt();
|
||||
offset = Math.max(offset, 0);
|
||||
int count = COUNT;
|
||||
if (jo.has("count")) count = jo.get("count").getAsInt();
|
||||
count = count < 0 ? COUNT : count;
|
||||
// Analyzer analyzer = new IKAnalyzer();
|
||||
// QueryParser queryParser = new QueryParser(type, analyzer);
|
||||
// Query rmQuery = queryParser.parse(keyword);
|
||||
|
||||
DirectoryReader indexReader = DirectoryReader.open(indexDir);
|
||||
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
|
||||
Query query = new TermRangeQuery(type, new BytesRef(from), new BytesRef(to), true, true);
|
||||
//Query query = new TermQuery(new Term(type, keyword));
|
||||
TopDocs docs = indexSearcher.search(query, 10);
|
||||
|
||||
ResultModel resultModel = null;
|
||||
if (docs.scoreDocs != null && docs.scoreDocs.length > 0) {
|
||||
resultModel = paginate(docs, indexReader, count,offset);
|
||||
|
||||
if (resultModel != null) {
|
||||
return resultModel.toString();
|
||||
}
|
||||
ResultModel noDataResult = new ResultModel();
|
||||
return noDataResult.toString();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
ByteArrayOutputStream bo = new ByteArrayOutputStream();
|
||||
e.printStackTrace(new PrintStream(bo));
|
||||
return bo.toString();
|
||||
}
|
||||
ResultModel noDataResult = new ResultModel();
|
||||
return noDataResult.toString();
|
||||
}
|
||||
/*
|
||||
{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0}
|
||||
*/
|
||||
public String searchByMultiTag(JsonObject jo) {
|
||||
try {
|
||||
int offset = OFFSET;
|
||||
if (jo.has("offset")) offset = jo.get("offset").getAsInt();
|
||||
offset = Math.max(offset, 0);
|
||||
int count = COUNT;
|
||||
if (jo.has("count")) count = jo.get("count").getAsInt();
|
||||
count = count < 0 ? COUNT : count;
|
||||
|
||||
//System.out.println(count+" "+offset);
|
||||
JsonArray parmas=jo.get("params").getAsJsonArray();
|
||||
parmas.get(0).getAsJsonObject().get("keyword");
|
||||
String[] type=new String[parmas.size()];
|
||||
String[] keyword=new String[parmas.size()];
|
||||
BooleanClause.Occur[] flags = new BooleanClause.Occur[parmas.size()];
|
||||
for(int i=0;i<parmas.size();i++){
|
||||
type[i]=parmas.get(i).getAsJsonObject().get("type").getAsString();
|
||||
keyword[i]=parmas.get(i).getAsJsonObject().get("keyword").getAsString();
|
||||
flags[i]=BooleanClause.Occur.SHOULD;
|
||||
//System.out.println(type[i]+" "+keyword[i]);
|
||||
}
|
||||
|
||||
DirectoryReader indexReader = DirectoryReader.open(indexDir);
|
||||
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
|
||||
Query query = MultiFieldQueryParser.parse(keyword, type, flags, new IKAnalyzer());
|
||||
TopDocs docs = indexSearcher.search(query, 10);
|
||||
ResultModel resultModel = null;
|
||||
if (docs.scoreDocs != null && docs.scoreDocs.length > 0) {
|
||||
resultModel = paginate(docs, indexReader, count,offset);
|
||||
if (resultModel != null) {
|
||||
return resultModel.toString();
|
||||
}
|
||||
ResultModel noDataResult = new ResultModel();
|
||||
return noDataResult.toString();
|
||||
}
|
||||
|
||||
}catch (Exception e){
|
||||
ByteArrayOutputStream bo = new ByteArrayOutputStream();
|
||||
e.printStackTrace(new PrintStream(bo));
|
||||
return bo.toString();
|
||||
}
|
||||
ResultModel noDataResult = new ResultModel();
|
||||
return noDataResult.toString();
|
||||
}
|
||||
|
||||
public ResultModel paginate(TopDocs docs, IndexReader reader,int count,int offset ) throws IOException {
|
||||
ResultModel resultModel = new ResultModel();
|
||||
resultModel.setContractCount(docs.totalHits.value);
|
||||
ScoreDoc[] scoreDocs = docs.scoreDocs;
|
||||
List<DoMeta> doMetaList = new ArrayList<>();
|
||||
|
||||
if (scoreDocs != null) {
|
||||
for (int i = offset; i < scoreDocs.length && i < offset+count; i++) {
|
||||
Document document = reader.document(scoreDocs[i].doc);
|
||||
JsonObject jo=new JsonObject();
|
||||
for (IndexableField in : document.getFields()) {
|
||||
String key=in.name();
|
||||
String val=in.stringValue();
|
||||
jo.addProperty(key,val);
|
||||
}
|
||||
DoMeta repometa=new DoMeta(jo);
|
||||
doMetaList.add(repometa);
|
||||
}
|
||||
}
|
||||
resultModel.setDoMetaList(doMetaList);
|
||||
return resultModel;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package org.bdware.datanet.searchengine;
|
||||
import com.google.gson.JsonObject;
|
||||
|
||||
import java.util.List;
|
||||
public class ResultModel {
|
||||
private List<DoMeta> DoMetaList;
|
||||
private Long DoCount;
|
||||
public ResultModel() {
|
||||
this.DoMetaList = null;
|
||||
this.DoCount = 0L;
|
||||
}
|
||||
|
||||
public List<DoMeta> getRepoMetaList() {
|
||||
return DoMetaList;
|
||||
}
|
||||
|
||||
public void setDoMetaList(List<DoMeta> metalist) {
|
||||
this.DoMetaList = metalist;
|
||||
Integer i= metalist.size();
|
||||
DoCount=i.longValue();
|
||||
}
|
||||
|
||||
public Long getRepoCount() {
|
||||
return DoCount;
|
||||
}
|
||||
|
||||
public void setContractCount(Long DoCount) {
|
||||
this.DoCount = DoCount;
|
||||
}
|
||||
public JsonObject toJson(){
|
||||
JsonObject jo=new JsonObject();
|
||||
//jo.addProperty("repoCount",repoCount);
|
||||
return jo;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
if(DoMetaList==null)
|
||||
return "{\"data\":[],\"total\":0}";
|
||||
else
|
||||
return "{" +
|
||||
"\"data\":" + DoMetaList.toString() +
|
||||
", \"total\":" + DoCount +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package org.bdware.datanet.searchengine;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import org.bdware.sc.ContractProcess;
|
||||
import org.bdware.sc.engine.JSONTool;
|
||||
|
||||
import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror;
|
||||
import wrp.jdk.nashorn.api.scripting.ScriptUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
|
||||
import javax.script.Invocable;
|
||||
import javax.script.ScriptException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public class SearchEngine{
|
||||
|
||||
private final Logger LOGGER = LogManager.getLogger(SearchEngine.class);
|
||||
public MetaIndex metaIndex;
|
||||
public SearchEngine(String Dirname){
|
||||
metaIndex=new MetaIndex(Dirname);
|
||||
}
|
||||
public SearchEngine(){
|
||||
metaIndex=new MetaIndex();
|
||||
}
|
||||
|
||||
|
||||
public static SearchEngine createObj(String Dirname){
|
||||
return new SearchEngine(Dirname);
|
||||
}
|
||||
public static SearchEngine createObj(){
|
||||
return new SearchEngine();
|
||||
}
|
||||
|
||||
public String test(){return "success";}
|
||||
|
||||
//params:
|
||||
//String type,String keyword,String count,String offset
|
||||
//String type,String keyword
|
||||
public String search(ScriptObjectMirror so){
|
||||
|
||||
if(JSONTool.convertMirrorToJson(so).isJsonObject()){
|
||||
JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject();
|
||||
System.out.println("search: "+jo);
|
||||
if(jo.get("count")==null) jo.addProperty("count",10);
|
||||
if(jo.get("offset")==null) jo.addProperty("offset",0);
|
||||
return metaIndex.search(jo);
|
||||
}
|
||||
return "failed";
|
||||
}
|
||||
|
||||
|
||||
|
||||
//根据时间范围查询 from :to
|
||||
public String searchByRange(ScriptObjectMirror so){
|
||||
|
||||
if(JSONTool.convertMirrorToJson(so).isJsonObject()){
|
||||
JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject();
|
||||
if(jo.get("count")==null) jo.addProperty("count",10);
|
||||
if(jo.get("offset")==null) jo.addProperty("offset",0);
|
||||
return metaIndex.searchByRange(jo);
|
||||
}
|
||||
return "failed";
|
||||
}
|
||||
public String search(String type,String keyword,int count,int offset){
|
||||
JsonObject jo =new JsonObject();
|
||||
jo.addProperty("type",type);
|
||||
jo.addProperty("keyword",keyword);
|
||||
jo.addProperty("offset",offset);
|
||||
jo.addProperty("count",count);
|
||||
return metaIndex.search(jo);
|
||||
}
|
||||
public String search(String type,String keyword,String count,String offset){
|
||||
JsonObject jo =new JsonObject();
|
||||
jo.addProperty("type",type);
|
||||
jo.addProperty("keyword",keyword);
|
||||
jo.addProperty("offset",offset);
|
||||
jo.addProperty("count",count);
|
||||
return metaIndex.search(jo);
|
||||
}
|
||||
|
||||
/*
|
||||
{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0}
|
||||
*/
|
||||
public String searchByMultiTag(ScriptObjectMirror so){
|
||||
JsonElement je=JSONTool.convertMirrorToJson(so);
|
||||
JsonObject jo =je.getAsJsonObject();
|
||||
return metaIndex.searchByMultiTag(jo);
|
||||
}
|
||||
|
||||
|
||||
|
||||
//{"doid":"xxx"}
|
||||
public String delete(ScriptObjectMirror so){
|
||||
|
||||
if(JSONTool.convertMirrorToJson(so).isJsonObject()){
|
||||
JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject();
|
||||
if(jo.get("doId")==null) return "failed";
|
||||
else{
|
||||
metaIndex.deleteRepo(jo.get("doId").getAsString());
|
||||
return "success";
|
||||
}
|
||||
}
|
||||
return "failed";
|
||||
}
|
||||
public String delete(String doId){
|
||||
metaIndex.deleteRepo(doId);
|
||||
return "success";
|
||||
}
|
||||
|
||||
public String search(String type, String keyword){
|
||||
JsonObject jo =new JsonObject();
|
||||
jo.addProperty("type",type);
|
||||
jo.addProperty("keyword",keyword);
|
||||
jo.addProperty("offset",0);
|
||||
jo.addProperty("count",10);
|
||||
return metaIndex.search(jo);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public int count(){
|
||||
return 0;
|
||||
}
|
||||
|
||||
//一个do或者一个do数组
|
||||
public String update(ScriptObjectMirror so) throws IOException, ParseException {
|
||||
JsonElement je=JSONTool.convertMirrorToJson(so);
|
||||
// if(je.isJsonArray()){
|
||||
// List<DoMeta> list=new ArrayList<>();
|
||||
// for(JsonElement s : je.getAsJsonArray()) {
|
||||
// DoMeta r = new DoMeta(s.getAsJsonObject());
|
||||
// list.add(r);
|
||||
// }
|
||||
// metaIndex.updateRepoList(list);
|
||||
// return "success";
|
||||
// }
|
||||
if(je.isJsonObject()){
|
||||
|
||||
DoMeta r=new DoMeta(je.getAsJsonObject());
|
||||
metaIndex.updateRepo(r);
|
||||
return "success";
|
||||
}
|
||||
else return "failed";
|
||||
}
|
||||
public String testScriptObjectMirror(ScriptObjectMirror so) throws IOException, ParseException {
|
||||
//JsonObject jo = (JsonObject) ScriptUtils.convert(so, JsonObject.class);
|
||||
JsonElement je=JSONTool.convertMirrorToJson(so);
|
||||
System.out.println(so.toString());
|
||||
System.out.println(so.getClass());
|
||||
System.out.println(je.getClass());
|
||||
System.out.println(je.toString());
|
||||
return "success";
|
||||
}
|
||||
public void addFunction(String function){
|
||||
try{
|
||||
ContractProcess.instance.engine.compileFunction(function,false);
|
||||
//ContractProcess.instance.engine.invokeFunction(function,args);
|
||||
|
||||
//ContractProcess.instance.engine
|
||||
}catch (ScriptException e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// public String updateList(JsonArray ja) throws IOException, ParseException {
|
||||
// List<RepoMeta> list=new ArrayList<>();
|
||||
// for(JsonElement s : ja.getAsJsonArray()){
|
||||
// RepoMeta r=new RepoMeta(s.getAsJsonObject());
|
||||
// list.add(r);
|
||||
// }
|
||||
// metaIndex.updateRepoList(list);
|
||||
// return "success";
|
||||
// }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
public class DebugMain {
|
||||
public static void main(String[]args){
|
||||
org.bdware.sc.debugger.DebugMain.main(null);
|
||||
}
|
||||
}
|
||||
54
datanet-search-engine-backend/src/test/java/GenerateDO.java
Normal file
54
datanet-search-engine-backend/src/test/java/GenerateDO.java
Normal file
@@ -0,0 +1,54 @@
|
||||
import com.google.gson.JsonObject;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.bdware.doip.audit.EndpointConfig;
|
||||
import org.bdware.doip.audit.client.AuditIrpClient;
|
||||
import org.bdware.doip.audit.config.FileStorage;
|
||||
import org.bdware.irp.exception.IrpClientException;
|
||||
import org.bdware.irp.stateinfo.StateInfoBase;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.PrintStream;
|
||||
|
||||
public class GenerateDO {
|
||||
static Logger LOGGER = Logger.getLogger(GenerateDO.class);
|
||||
AuditIrpClient client;
|
||||
JsonObject otherConfig;
|
||||
|
||||
@Before
|
||||
public void init() throws FileNotFoundException {
|
||||
FileStorage fs = new FileStorage("./input/auditrepo.json");
|
||||
EndpointConfig config = fs.loadAsEndpointConfig();
|
||||
config.repoName = null;
|
||||
otherConfig = fs.load();
|
||||
client = new AuditIrpClient(config);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
FileOutputStream fout = new FileOutputStream("./input/doi.txt",true);
|
||||
PrintStream ps = new PrintStream(fout);
|
||||
StateInfoBase base = new StateInfoBase();
|
||||
base.handleValues = new JsonObject();
|
||||
base.handleValues.addProperty("repoId", otherConfig.get("targetRepo").getAsString());
|
||||
base.handleValues.addProperty("timestamp", System.currentTimeMillis());
|
||||
try {
|
||||
String doid = client.register(base);
|
||||
if (doid != null) ps.println(doid);
|
||||
String format = "http://%s/SCIDE/SCManager?action=executeContract&contractID=SearchEngine&operation=testAddDo&arg=";
|
||||
String argStr = "{\"doId\":\"%s\",\"name\":\"repofile\",\"type\":\"text\",\"description\":\"这是一个模拟DO\",\"owner\":\"xx省yy市zz部门\",\"createTime\":%d}";
|
||||
format += argStr;
|
||||
String seURI = otherConfig.get("seURI").getAsString();
|
||||
long createTime = System.currentTimeMillis();
|
||||
long l = (long) (Math.random() * 1000D * 3600D * 24D);
|
||||
String data = String.format(format, seURI, doid, createTime - l);
|
||||
System.out.println(data);
|
||||
} catch (IrpClientException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
fout.close();
|
||||
ps.close();
|
||||
}
|
||||
}
|
||||
26
datanet-search-engine-backend/src/test/java/GraphDBTEst.java
Normal file
26
datanet-search-engine-backend/src/test/java/GraphDBTEst.java
Normal file
@@ -0,0 +1,26 @@
|
||||
import com.google.gson.JsonObject;
|
||||
import org.bdware.datanet.searchengine.MetaGraph;
|
||||
import org.junit.Test;
|
||||
|
||||
public class GraphDBTEst {
|
||||
@Test
|
||||
public void Test(){
|
||||
MetaGraph m=new MetaGraph("test");
|
||||
JsonObject do1=new JsonObject();
|
||||
do1.addProperty("id", "1");
|
||||
JsonObject do2=new JsonObject();
|
||||
do2.addProperty("id","2");
|
||||
m.createDoNode("001",do1);
|
||||
m.createDoNode("002",do2);
|
||||
JsonObject jo=new JsonObject();
|
||||
jo.addProperty("id1","001");
|
||||
jo.addProperty("id2","002");
|
||||
jo.addProperty("key1","stu_no");
|
||||
jo.addProperty("key2","student_no");
|
||||
m.createRelationDo2Do(jo);
|
||||
m.iterDO();
|
||||
m.iterRealField();
|
||||
m.iterStandardField();
|
||||
|
||||
}
|
||||
}
|
||||
69
datanet-search-engine-backend/src/test/java/luceneTest.java
Normal file
69
datanet-search-engine-backend/src/test/java/luceneTest.java
Normal file
@@ -0,0 +1,69 @@
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
|
||||
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
|
||||
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
|
||||
import org.junit.Test;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||
//import org.wltea.analyzer.lucene.IKAnalyzer;
|
||||
|
||||
import java.io.StringReader;
|
||||
|
||||
public class luceneTest {
|
||||
@Test
|
||||
public void StandardAnalyzerTest() throws Exception{
|
||||
//英文测试
|
||||
String text="An IndexWriter creaters and maintains an index.";
|
||||
Analyzer analyzer=new StandardAnalyzer();
|
||||
displayTokens(analyzer,text);
|
||||
//中文测试
|
||||
String text1="我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。";
|
||||
displayTokens(analyzer,text1);
|
||||
}
|
||||
/**
|
||||
58 * IKAnalyzerTest分词法测试,对中文支持很好,词库分词
|
||||
59 * @throws Exception
|
||||
60 */
|
||||
@Test
|
||||
public void IKAnalyzerTest() throws Exception{
|
||||
//英文测试
|
||||
String text="An IndexWriter creaters and maintains an index.";
|
||||
// Analyzer analyzer=new IKAnalyzer();
|
||||
Analyzer analyzer=new StandardAnalyzer();
|
||||
displayTokens(analyzer,text);
|
||||
//中文测试
|
||||
String text1="我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。";
|
||||
displayTokens(analyzer,text1);
|
||||
}
|
||||
public static void displayTokens(Analyzer analyzer, String text) throws Exception {
|
||||
System.out.println("当前使用的分词器:" + analyzer.getClass().getName());
|
||||
//分词流,即将对象分词后所得的Token在内存中以流的方式存在,也说是说如果在取得Token必须从TokenStream中获取,而分词对象可以是文档文本,也可以是查询文本。
|
||||
TokenStream tokenStream = analyzer.tokenStream("content", new StringReader(text));
|
||||
//表示token的首字母和尾字母在原文本中的位置。比如I'm的位置信息就是(0,3),需要注意的是startOffset与endOffset的差值并不一定就是termText.length(),
|
||||
//因为可能term已经用stemmer或者其他过滤器处理过;
|
||||
OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class);
|
||||
//这个有点特殊,它表示tokenStream中的当前token与前一个token在实际的原文本中相隔的词语数量,用于短语查询。比如: 在tokenStream中[2:a]的前一个token是[1:I'm ],
|
||||
//它们在原文本中相隔的词语数是1,则token="a"的PositionIncrementAttribute值为1;
|
||||
PositionIncrementAttribute positionIncrementAttribute = tokenStream.addAttribute(PositionIncrementAttribute.class);
|
||||
|
||||
CharTermAttribute charTermAttribute= tokenStream.addAttribute(CharTermAttribute.class);
|
||||
|
||||
//表示token词典类别信息,默认为“Word”,比如I'm就属于<APOSTROPHE>,有撇号的类型;
|
||||
TypeAttribute typeAttribute = tokenStream.addAttribute(TypeAttribute.class);
|
||||
tokenStream.reset();
|
||||
|
||||
int position = 0;
|
||||
while (tokenStream.incrementToken()) {
|
||||
int increment = positionIncrementAttribute.getPositionIncrement();
|
||||
if(increment > 0) {
|
||||
position = position + increment;
|
||||
}
|
||||
int startOffset = offsetAttribute.startOffset();
|
||||
int endOffset = offsetAttribute.endOffset();
|
||||
String term ="输出结果为:"+ charTermAttribute.toString();
|
||||
System.out.println("第"+position+"个分词,分词内容是:[" + term + "]" + ",分词内容的开始结束位置为:(" + startOffset + "-->" + endOffset + "),类型是:" + typeAttribute.type());
|
||||
}
|
||||
tokenStream.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import wrp.jdk.nashorn.api.scripting.NashornScriptEngine;
|
||||
import wrp.jdk.nashorn.api.scripting.ScriptUtils;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.bdware.sc.boundry.JavaScriptEntry;
|
||||
import org.bdware.sc.engine.JSONTool;
|
||||
import org.junit.Test;
|
||||
import org.bdware.datanet.searchengine.SearchEngine;
|
||||
import wrp.jdk.nashorn.api.scripting.NashornScriptEngine;
|
||||
import wrp.jdk.nashorn.api.scripting.NashornScriptEngineFactory;
|
||||
import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror;
|
||||
import wrp.jdk.nashorn.internal.objects.Global;
|
||||
import wrp.jdk.nashorn.internal.runtime.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
public class searchEnginetest {
|
||||
public static String dbname="MetaIndex01";
|
||||
@Test
|
||||
public void test2() throws IOException, ParseException{
|
||||
System.out.println("hello");
|
||||
//1642696178922
|
||||
System.out.println(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void addDo() {
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
//SearchEngine se=new SearchEngine();
|
||||
try{
|
||||
engine.eval("var r1 = {\"url\":\"39.106.6.6:3306\",\"user\":\"loushuai\",\"password\":\"loushuai\",\"dbname\":\"newele\",\"tablename\":\"data\",\"createTime\":\"1641363811328\",\"owner\":\"zzz\",\"description\":\"这是一段说明\",\"schema\":\"{}\",\"prefix\":\"aibd.govdata.tj\",\"type\":\"SQL\",\"doId\":\"aibd.govdata.tj/DBRepo/do.newele.data\"};");
|
||||
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r1"));
|
||||
System.out.println(se.update((ScriptObjectMirror) engine.eval("r1")));
|
||||
//System.out.println(se.search("info","产品"));
|
||||
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void addDo2() {
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
//SearchEngine se=new SearchEngine();
|
||||
try{
|
||||
engine.eval("var r1 = {doId:\"AIBD/doip.Repo10101\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。\"};");
|
||||
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r1"));
|
||||
System.out.println(se.update((ScriptObjectMirror) engine.eval("r1")));
|
||||
|
||||
engine.eval("var r2 = {doId:\"AIBD/doip.Repo10102\", type:\"readme\", createTime:\"1842696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。\"};");
|
||||
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r2"));
|
||||
System.out.println(se.update((ScriptObjectMirror) engine.eval("r2")));
|
||||
//System.out.println(se.search("info","产品"));
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
@Test
|
||||
public void addDoList()
|
||||
{
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
try{
|
||||
engine.eval("var j1 = [{doId:\"AIBD/doip.Repo10101\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障……\"}," +
|
||||
"{doId:\"AIBD/doip.Repo10102\", type:\"readme\", createTime:\"1642696198922\",info:\"你们这款产品底层逻辑是打通信息屏障……\"}];");
|
||||
se.update((ScriptObjectMirror) engine.eval("j1"));
|
||||
System.out.println(se.search("info","产品"));
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
@Test
|
||||
public void searchRepoTest() throws IOException, ParseException {
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
try{
|
||||
engine.eval("var arg ={type:\"doId\",keyword:\"AIBD/doip.Repo\"};");
|
||||
System.out.println(se.search((ScriptObjectMirror) engine.eval("arg")));
|
||||
engine.eval("var arg1 ={type:\"owner\",keyword:\"zzz\"};");
|
||||
System.out.println(se.search((ScriptObjectMirror) engine.eval("arg1")));
|
||||
//System.out.println(se.search("info","产品",5,0));
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@Test
|
||||
public void searchByRangeTest() throws IOException, ParseException {
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
try{
|
||||
engine.eval("var arg ={type:\"createTime\",from:\"1642696178922\",to:\"1942696178922\"};");
|
||||
//engine.eval("var arg ={type:\"createTime\",keyword:\"createTime\",from:\"1642696178922\",to:\"1642696178922\"};");
|
||||
System.out.println(se.searchByRange ((ScriptObjectMirror) engine.eval("arg")));
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0}
|
||||
|
||||
@Test
|
||||
public void searchByMultiTagTest() {
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
try{
|
||||
// engine.eval("var r1 = {doId:\"repo1\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。\"};");
|
||||
// System.out.println(se.update((ScriptObjectMirror) engine.eval("r1")));
|
||||
//
|
||||
// engine.eval("var r2 = {doId:\"repo2\", type:\"sql\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。\"};");
|
||||
// System.out.println(se.update((ScriptObjectMirror) engine.eval("r2")));
|
||||
|
||||
engine.eval("var arg2 ={type:\"info\",keyword:\"我们\",count:10,offset:0};");
|
||||
System.out.println(se.search ((ScriptObjectMirror) engine.eval("arg2")));
|
||||
|
||||
|
||||
engine.eval("var arg1 ={params:[{type:\"doId\",keyword:\"AIBDdoip.Repo1001\"}],count:10,offset:0};");
|
||||
System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg1")));
|
||||
|
||||
engine.eval("var arg ={params:[{type:\"info\",keyword:\"我们\"},{type:\"doId\",keyword:\"repo\"}],count:10,offset:0};");
|
||||
System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg")));
|
||||
|
||||
engine.eval("var arg ={\n" +
|
||||
" params: [{ type: 'doId', keyword: 'd' }],\n" +
|
||||
" count: 100,\n" +
|
||||
" offset: 0,\n" +
|
||||
" };");
|
||||
System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg")));
|
||||
|
||||
|
||||
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deleteTest(){
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
try{
|
||||
|
||||
engine.eval("var r1 = {doId:\"AIBD/doip.Repo1001\"};");
|
||||
System.out.println(se.delete((ScriptObjectMirror)engine.eval("r1")));
|
||||
//se.delele((ScriptObjectMirror)engine.eval("doid"));
|
||||
System.out.println(se.search("info","产品"));
|
||||
System.out.println(se.search("doId","AIBD/doip.Repo1002"));
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
@Test
|
||||
public void jsTest(){
|
||||
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
|
||||
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
|
||||
SearchEngine se=SearchEngine.createObj(dbname);
|
||||
try{
|
||||
System.out.println(engine.eval("var car = {name:\"Fiat\", model:500, color:\"white\"};"));
|
||||
System.out.println(engine.eval("car").getClass());
|
||||
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("car"));
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user