This commit is contained in:
PhiTer9426
2022-05-03 21:07:23 +08:00
parent 6152edc1b6
commit 641d6cda65
46 changed files with 5026 additions and 2 deletions

View File

@@ -0,0 +1,16 @@
package org.bdware.datanet.searchengine;
import com.google.gson.JsonObject;
//RepoMeta要求是一个jsonObject要求必须要有一个doi
public class DoMeta {
JsonObject metadata;
DoMeta(JsonObject jo){
metadata=jo.deepCopy();
}
@Override
public String toString() {
return metadata.toString();
}
}

View File

@@ -0,0 +1,333 @@
package org.bdware.datanet.searchengine;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import org.bdware.sc.engine.JSONTool;
import org.junit.Test;
import org.neo4j.graphdb.*;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.index.Index;
import org.neo4j.graphdb.traversal.Evaluators;
import org.neo4j.graphdb.traversal.TraversalDescription;
import java.io.File;
import java.util.Iterator;
import java.util.Map;
public class MetaGraph {
// private static final String DB_PATH = "testgraph.db";
private static String DB_PATH="./MetaGraph.db";
// private static enum LABEL {
// DO, REAL, STANDARD
// }
private GraphDatabaseService graphDB;
public MetaGraph(String name){
DB_PATH="./"+name+".db";
graphDB = new GraphDatabaseFactory().newEmbeddedDatabase(new File(DB_PATH));
}
//DO类型
/*
{
"doid":"xxx",
"maskinfo":{
"stu_no":"@int(5,10)",
"name":"cname"
}
}
*/
public void createDoNode(String doid,JsonObject jo) {
Transaction tx = graphDB.beginTx();
Node Do = graphDB.createNode(new NodeLabel("DO"));//命名规则
Do.setProperty("doid",doid);
Iterator iter = jo.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry entry = (Map.Entry) iter.next();
Do.setProperty(entry.getKey().toString(),entry.getValue().toString());
}
tx.success();
tx.close();
//配置do的mask
}
public void iterDO() {
Transaction tx = graphDB.beginTx();
ResourceIterator<Node> iterator = graphDB.findNodes(new NodeLabel("DO"));
for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
Node node = it.next();
String doid=(String)node.getProperty("doid");
String id=(String)node.getProperty("id");
System.out.println(doid + " " + id+ " ");
}
tx.success();
tx.close();
}
//param:{"doid1":"xxx","doid2":"yyy","key1":"studentid","key2":"studentid"}
public void createRelationDo2Do(JsonObject jo){
Transaction tx = graphDB.beginTx();
String doid1=jo.get("id1").getAsString();
String doid2=jo.get("id2").getAsString();
String key1=jo.get("key1").getAsString();
String key2=jo.get("key2").getAsString();
Node node1=graphDB.findNode(new NodeLabel("DO"),"doid",doid1);
Node node2=graphDB.findNode(new NodeLabel("DO"),"doid",doid2);
System.out.println(jo);
node1.createRelationshipTo(node2, new RelationType(jo.toString()));
tx.success();
tx.close();
}
//真实标签类型
public void createRealNode(String key,String value) {
Transaction tx = graphDB.beginTx();
Node realField = graphDB.createNode(new NodeLabel("realField"));//命名规则
realField.setProperty("key",key);
realField.setProperty("value",value);
tx.success();
tx.close();
}
//标准标签
public void createStandardNode(String key,String value) {
Transaction tx = graphDB.beginTx();
Node standardField = graphDB.createNode(new NodeLabel("standardField"));//命名规则
standardField.setProperty("key",key);
standardField.setProperty("value",value);
tx.success();
tx.close();
}
public void iterRealField() {
Transaction tx = graphDB.beginTx();
ResourceIterator<Node> iterator = graphDB.findNodes(new NodeLabel("DO"));
for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
Node node = it.next();
String doid=(String)node.getProperty("doid");
String id=(String)node.getProperty("id");
System.out.println(doid + " " + id+ " ");
}
tx.success();
tx.close();
}
public void iterStandardField() {
Transaction tx = graphDB.beginTx();
ResourceIterator<Node> iterator = graphDB.findNodes(new NodeLabel("DO"));
for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
Node node = it.next();
String doid=(String)node.getProperty("doid");
String id=(String)node.getProperty("id");
System.out.println(doid + " " + id+ " ");
}
tx.success();
tx.close();
}
// public void init() {
// graphDB = new GraphDatabaseFactory().newEmbeddedDatabase(new File(DB_PATH));
// }
// /**创建数据*/
// public void create() {
// Transaction tx = graphDB.beginTx();
//
// Node student = graphDB.createNode(new CaseLabel("STUDENT"));//命名规则
// student.setProperty("name", "@student");//数据类型命名规则
// student.setProperty("id", "@int(2001210000,2001210999)");
//
// Node course = graphDB.createNode(new CaseLabel("COURSE"));//命名规则
// course.setProperty("name", "[枚举or读文件]");//数据类型命名规则
// course.setProperty("id", "@int(10000,10999)");
//
// student.createRelationshipTo(course, new RelTypes("CRIME"));
//
//
//
// Node case1 = graphDB.createNode(new CaseLabel("CASEINFO"));
// case1.setProperty("name", "案件1");
// case1.setProperty("address", "南山");
//
// Node case2 = graphDB.createNode(new CaseLabel("CASEINFO"));
// case2.setProperty("name", "案件2");
// case2.setProperty("address", "福田");
//
// Node case3 = graphDB.createNode(new CaseLabel("CASEINFO"));
// case3.setProperty("name", "案件3");
// case3.setProperty("address", "龙华");
//
// Node userA = graphDB.createNode(new CaseLabel("PERSON"));
// userA.setProperty("name", "A");
// userA.setProperty("idcard", "150302198012228239");
// userA.setProperty("tel", "13685246639");
//
// Node userB = graphDB.createNode(new CaseLabel("PERSON"));
// userB.setProperty("name", "B");
// userB.setProperty("idcard", "370634199208304929");
// userB.setProperty("tel", "13885246670");
//
// Node userC = graphDB.createNode(new CaseLabel("PERSON"));
// userC.setProperty("name", "C");
// userC.setProperty("idcard", "430721198802065735");
// userC.setProperty("tel", "13966704782");
//
// Node userD = graphDB.createNode(new CaseLabel("PERSON"));
// userD.setProperty("name", "D");
// userD.setProperty("idcard", "522730198707118747");
// userD.setProperty("tel", "13670478962");
//
// Node userE = graphDB.createNode(new CaseLabel("PERSON"));
// userE.setProperty("name", "E");
// userE.setProperty("idcard", "622926198609158032");
// userE.setProperty("tel", "13047829667");
//
// Node userF = graphDB.createNode(new CaseLabel("PERSON"));
// userF.setProperty("name", "F");
// userF.setProperty("idcard", "500114197706138305");
// userF.setProperty("tel", "13478296670");
//
// Node userG = graphDB.createNode(new CaseLabel("PERSON"));
// userG.setProperty("name", "G");
// userG.setProperty("idcard", "500114106138305152");
// userG.setProperty("tel", "13476670156");
//
// Node userH = graphDB.createNode(new CaseLabel("PERSON"));
// userH.setProperty("name", "H");
// userH.setProperty("idcard", "500114197704751236");
// userH.setProperty("tel", "13296156670");
//
// case1.createRelationshipTo(userA, new RelTypes("CRIME"));
// case1.createRelationshipTo(userB, new RelTypes("CRIME"));
// case1.createRelationshipTo(userC, new RelTypes("CRIME"));
// case1.createRelationshipTo(userD, new RelTypes("CRIME"));
//
// case2.createRelationshipTo(userB, new RelTypes("CRIME"));
// case2.createRelationshipTo(userC, new RelTypes("CRIME"));
// case2.createRelationshipTo(userE, new RelTypes("CRIME"));
//// case2.createRelationshipTo(userD, RelTypes.CRIME);
//
//
// case3.createRelationshipTo(userF, new RelTypes("CRIME"));
// case3.createRelationshipTo(userG, new RelTypes("CRIME"));
// case3.createRelationshipTo(userH, new RelTypes("CRIME"));
//
// tx.success();
// tx.close();
//
// }
//
// public void search(String username) {
// Transaction tx = graphDB.beginTx();
// System.out.println(graphDB.getAllNodes().stream().findFirst());
// Node startNode = graphDB.findNode( "name", username);
//
// Iterable<Relationship> iterable = startNode
// .getRelationships(new RelTypes("CRIME"), Direction.INCOMING);
// for (Relationship r : iterable) {
//
// Node node = r.getStartNode();
// long id = node.getId();
// String name = (String)node.getProperty("name");
//// String idcard = (String)node.getProperty("idcard");
//// String tel = (String)node.getProperty("tel");
//
// System.out.println(id + " " + name+ " ");
// }
//
// tx.success();
// tx.close();
// }
//
// /**遍历查找**/
// public void searchUser(String username) {
// Transaction tx = graphDB.beginTx();
//
// Node startNode = graphDB.findNode(new CaseLabel("PERSON"), "name", username);
//// Node startNode = graphDB.findNode(new CaseLabel("PERSON"), "name", "B");
//
// TraversalDescription td = graphDB.traversalDescription();
// td.relationships(new RelTypes("CRIME"), Direction.OUTGOING);//沿着关系 TONG
// td.depthFirst();//设置深度优先
//// td.evaluator(Evaluators.excludeStartPosition());
//
// //从开始节点开始
//
// Iterable<Node> it = td.traverse(startNode).nodes();
// for (Node node : it) {
// long id = node.getId();
// if( node.hasLabel( new CaseLabel("PERSON") )) {
// String name = (String)node.getProperty("name");
// String idcard = (String)node.getProperty("idcard");
// String tel = (String)node.getProperty("tel");
//
// System.out.println(id + " " + name+ " " + idcard + " " + tel);
// } else {
//// String name = (String)node.getProperty("name");
//// String address = (String)node.getProperty("address");
//// System.out.println(id + " " + name+ " " + address);
// }
// }
//
// tx.success();
// tx.close();
// }
//
//
// /**查询所有作案人员信息*/
// public void searchAllPerson() {
//
//
//
// Transaction tx = graphDB.beginTx();
//
// ResourceIterator<Node> iterator = graphDB.findNodes(new CaseLabel("PERSON"));
// for (ResourceIterator<Node> it = iterator; it.hasNext(); ) {
// Node node = it.next();
// long id = node.getId();
// String name = (String)node.getProperty("name");
// String idcard = (String)node.getProperty("idcard");
// String tel = (String)node.getProperty("tel");
//
// System.out.println(id + " " + name+ " " + idcard + " " + tel);
// }
//
// tx.success();
// tx.close();
// }
}
class RelationType implements RelationshipType {
private String name;
public RelationType(String name) {
this.name = name;
}
public String name(){
return name;
}
}
class NodeLabel implements Label {
private String name;
public NodeLabel(String name) {
this.name = name;
}
public String name(){
return name;
}
}

View File

@@ -0,0 +1,341 @@
package org.bdware.datanet.searchengine;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.bdware.sc.boundry.utils.RocksDBUtil;
import org.rocksdb.RocksIterator;
import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.*;
import java.nio.file.Paths;
import java.util.*;
public class MetaIndex {
private final Logger LOGGER = LogManager.getLogger(MetaIndex.class);
public FSDirectory indexDir;
public boolean isEmpty = false;//描述当前repo的文件是否存在
private IndexWriter indexWriter;
private static final int COUNT=20;
private static final int OFFSET=0;
MetaIndex(String name){
initIndex(name);
}
MetaIndex(){
initIndex("SearchEngineIndex"+System.currentTimeMillis());
}
public void initIndex(String Dirname) {
try {
File dir = new File(Dirname);
if (!dir.exists()) {
LOGGER.info("make metaIndex dir ");
dir.mkdirs();
isEmpty = true;
}
indexDir = FSDirectory.open(Paths.get(dir.toURI()));
Analyzer analyzer = new IKAnalyzer();
indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(analyzer));
indexWriter.commit();
} catch (Exception e) {
e.printStackTrace();
}
}
public void indexRepo(DoMeta meta) throws IOException {
try {
//IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig());
Document document = new Document();
// System.out.println(meta.metadata);
for (Map.Entry entry : meta.metadata.entrySet()) {
String key=entry.getKey().toString();
System.out.println(entry.getValue());
String val=entry.getValue().toString().substring(1,entry.getValue().toString().length()-1);
System.out.println(key+" "+val);
//if(key.equals("doid")||key.equals("owner"))
// document.add(new StringField(key,val, Field.Store.YES));
//范围类型
if(key.equals("createTime")||key.equals("createDate")){
document.add(new StringField(key,val,Field.Store.YES));
}
else if(key.equals("doId")){
document.add(new StringField(key,val,Field.Store.YES));
//document.add(new LongPoint(key,Long.valueOf(val)),Field.Store.YES);
}
else{
document.add(new TextField(key,val, Field.Store.YES));
}
//System.out.println("eee"+entry.getKey().toString()+" "+entry.getValue().toString().substring(1,entry.getValue().toString().length()-1));
}
indexWriter.addDocument(document);
indexWriter.commit();
isEmpty = false;
}
catch (Exception e) {
e.printStackTrace();
}
}
public void updateRepoList(List<DoMeta> DoRepos)
throws IOException, ParseException {
IndexReader indexReader = null;
for (DoMeta r : DoRepos) {
String doId=r.metadata.get("doId").toString();
if(isEmpty){
indexRepo(r);
LOGGER.info("--> index when dir is not empty");
LOGGER.info("contract " + doId + " --> actually to index");
}
else{
try {
indexReader = DirectoryReader.open(indexDir);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
LOGGER.debug( doId+ "--> try to index");
//Query query = new TermQuery(new Term("doid", "AIBD/doip.Repo1001"));
Query query = new TermQuery(new Term("doId", doId));
TopDocs docs = indexSearcher.search(query, 10);
LOGGER.debug(docs.scoreDocs);
if(docs.scoreDocs == null || docs.scoreDocs.length == 0) {
indexRepo(r);
LOGGER.info("--> index when dir is not empty");
LOGGER.info("contract " + doId + " --> actually to index");
}
docs = indexSearcher.search(query, 10);
} catch (Exception e) {
LOGGER.warn("getting index failed! " + e.getMessage());
}
}
}
if (null != indexReader) {
indexReader.close();
}
}
public void updateRepo(DoMeta DoRepo)
throws IOException, ParseException {
IndexReader indexReader = null;
String doId=DoRepo.metadata.get("doId").getAsString();
if(isEmpty){
indexRepo(DoRepo);
LOGGER.info("--> index when dir is not empty");
LOGGER.info("contract " + doId + " --> actually to index");
}
else{
try {
indexReader = DirectoryReader.open(indexDir);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
LOGGER.debug( doId+ "--> try to index");
Query query = new TermQuery(new Term("doId", doId));
TopDocs docs = indexSearcher.search(query, 10);
LOGGER.debug(docs.scoreDocs);
System.out.println("bb"+docs.scoreDocs.length);
if(docs.scoreDocs == null || docs.scoreDocs.length == 0) {
indexRepo(DoRepo);
LOGGER.info("--> index when dir is not empty");
LOGGER.info("--> index when dir is not empty");
LOGGER.info("contract " + doId + " --> actually to index");
}
docs = indexSearcher.search(query, 10);
} catch (Exception e) {
LOGGER.warn("getting index failed! " + e.getMessage());
}
}
if (null != indexReader) {
indexReader.close();
}
}
public void deleteRepo(String doId) {
try {
IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig());
Query query=new TermQuery( new Term("doId",doId));
System.out.println("query"+query.toString());
indexWriter.deleteDocuments(query);
indexWriter.commit();
indexWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
//模糊搜索
public String search(JsonObject jo) {
try {
if (!jo.has("type")) {
return "missing arguments";
}
String type=jo.get("type").getAsString();
if (!jo.has("keyword")) return "missing arguments: keyword";
int offset=OFFSET;
if (jo.has("offset")) offset= jo.get("offset").getAsInt();
offset= Math.max(offset, 0);
int count=COUNT;
if (jo.has("count")) count= jo.get("count").getAsInt();
count=count<0?COUNT:count;
String keyword = jo.get("keyword").getAsString();
System.out.println(type+" "+keyword);
// Analyzer analyzer = new IKAnalyzer();
// QueryParser queryParser = new QueryParser(type, analyzer);
// Query rmQuery = queryParser.parse(keyword);
DirectoryReader indexReader = DirectoryReader.open(indexDir);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Query query = new TermQuery(new Term(type, keyword));
TopDocs docs = indexSearcher.search(query, 10);
System.out.println("length "+docs.scoreDocs.length);
ResultModel resultModel = null;
if (docs.scoreDocs != null && docs.scoreDocs.length > 0)
resultModel = paginate(docs, indexReader,count,offset);
if (resultModel != null) {
return resultModel.toString();
}
ResultModel noDataResult=new ResultModel();
return noDataResult.toString();
}
catch (Exception e) {
ByteArrayOutputStream bo = new ByteArrayOutputStream();
e.printStackTrace(new PrintStream(bo));
return bo.toString();
}
}
//按照范围查询
public String searchByRange(JsonObject jo) {
try {
if (!jo.has("type")) {
return "missing arguments";
}
//if (!jo.has("keyword")) return "missing arguments: keyword";
if (!jo.has("from")) return "missing arguments: from";
if (!jo.has("to")) return "missing arguments: to";
String type = jo.get("type").getAsString();
//String keyword = jo.get("keyword").getAsString();
String from = jo.get("from").getAsString();
String to = jo.get("to").getAsString();
int offset = OFFSET;
if (jo.has("offset")) offset = jo.get("offset").getAsInt();
offset = Math.max(offset, 0);
int count = COUNT;
if (jo.has("count")) count = jo.get("count").getAsInt();
count = count < 0 ? COUNT : count;
// Analyzer analyzer = new IKAnalyzer();
// QueryParser queryParser = new QueryParser(type, analyzer);
// Query rmQuery = queryParser.parse(keyword);
DirectoryReader indexReader = DirectoryReader.open(indexDir);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Query query = new TermRangeQuery(type, new BytesRef(from), new BytesRef(to), true, true);
//Query query = new TermQuery(new Term(type, keyword));
TopDocs docs = indexSearcher.search(query, 10);
ResultModel resultModel = null;
if (docs.scoreDocs != null && docs.scoreDocs.length > 0) {
resultModel = paginate(docs, indexReader, count,offset);
if (resultModel != null) {
return resultModel.toString();
}
ResultModel noDataResult = new ResultModel();
return noDataResult.toString();
}
} catch (Exception e) {
ByteArrayOutputStream bo = new ByteArrayOutputStream();
e.printStackTrace(new PrintStream(bo));
return bo.toString();
}
ResultModel noDataResult = new ResultModel();
return noDataResult.toString();
}
/*
{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0}
*/
public String searchByMultiTag(JsonObject jo) {
try {
int offset = OFFSET;
if (jo.has("offset")) offset = jo.get("offset").getAsInt();
offset = Math.max(offset, 0);
int count = COUNT;
if (jo.has("count")) count = jo.get("count").getAsInt();
count = count < 0 ? COUNT : count;
//System.out.println(count+" "+offset);
JsonArray parmas=jo.get("params").getAsJsonArray();
parmas.get(0).getAsJsonObject().get("keyword");
String[] type=new String[parmas.size()];
String[] keyword=new String[parmas.size()];
BooleanClause.Occur[] flags = new BooleanClause.Occur[parmas.size()];
for(int i=0;i<parmas.size();i++){
type[i]=parmas.get(i).getAsJsonObject().get("type").getAsString();
keyword[i]=parmas.get(i).getAsJsonObject().get("keyword").getAsString();
flags[i]=BooleanClause.Occur.SHOULD;
//System.out.println(type[i]+" "+keyword[i]);
}
DirectoryReader indexReader = DirectoryReader.open(indexDir);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Query query = MultiFieldQueryParser.parse(keyword, type, flags, new IKAnalyzer());
TopDocs docs = indexSearcher.search(query, 10);
ResultModel resultModel = null;
if (docs.scoreDocs != null && docs.scoreDocs.length > 0) {
resultModel = paginate(docs, indexReader, count,offset);
if (resultModel != null) {
return resultModel.toString();
}
ResultModel noDataResult = new ResultModel();
return noDataResult.toString();
}
}catch (Exception e){
ByteArrayOutputStream bo = new ByteArrayOutputStream();
e.printStackTrace(new PrintStream(bo));
return bo.toString();
}
ResultModel noDataResult = new ResultModel();
return noDataResult.toString();
}
public ResultModel paginate(TopDocs docs, IndexReader reader,int count,int offset ) throws IOException {
ResultModel resultModel = new ResultModel();
resultModel.setContractCount(docs.totalHits.value);
ScoreDoc[] scoreDocs = docs.scoreDocs;
List<DoMeta> doMetaList = new ArrayList<>();
if (scoreDocs != null) {
for (int i = offset; i < scoreDocs.length && i < offset+count; i++) {
Document document = reader.document(scoreDocs[i].doc);
JsonObject jo=new JsonObject();
for (IndexableField in : document.getFields()) {
String key=in.name();
String val=in.stringValue();
jo.addProperty(key,val);
}
DoMeta repometa=new DoMeta(jo);
doMetaList.add(repometa);
}
}
resultModel.setDoMetaList(doMetaList);
return resultModel;
}
}

View File

@@ -0,0 +1,48 @@
package org.bdware.datanet.searchengine;
import com.google.gson.JsonObject;
import java.util.List;
public class ResultModel {
private List<DoMeta> DoMetaList;
private Long DoCount;
public ResultModel() {
this.DoMetaList = null;
this.DoCount = 0L;
}
public List<DoMeta> getRepoMetaList() {
return DoMetaList;
}
public void setDoMetaList(List<DoMeta> metalist) {
this.DoMetaList = metalist;
Integer i= metalist.size();
DoCount=i.longValue();
}
public Long getRepoCount() {
return DoCount;
}
public void setContractCount(Long DoCount) {
this.DoCount = DoCount;
}
public JsonObject toJson(){
JsonObject jo=new JsonObject();
//jo.addProperty("repoCount",repoCount);
return jo;
}
@Override
public String toString() {
if(DoMetaList==null)
return "{\"data\":[],\"total\":0}";
else
return "{" +
"\"data\":" + DoMetaList.toString() +
", \"total\":" + DoCount +
'}';
}
}

View File

@@ -0,0 +1,182 @@
package org.bdware.datanet.searchengine;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import org.bdware.sc.ContractProcess;
import org.bdware.sc.engine.JSONTool;
import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror;
import wrp.jdk.nashorn.api.scripting.ScriptUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.queryparser.classic.ParseException;
import javax.script.Invocable;
import javax.script.ScriptException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class SearchEngine{
private final Logger LOGGER = LogManager.getLogger(SearchEngine.class);
public MetaIndex metaIndex;
public SearchEngine(String Dirname){
metaIndex=new MetaIndex(Dirname);
}
public SearchEngine(){
metaIndex=new MetaIndex();
}
public static SearchEngine createObj(String Dirname){
return new SearchEngine(Dirname);
}
public static SearchEngine createObj(){
return new SearchEngine();
}
public String test(){return "success";}
//params:
//String type,String keyword,String count,String offset
//String type,String keyword
public String search(ScriptObjectMirror so){
if(JSONTool.convertMirrorToJson(so).isJsonObject()){
JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject();
System.out.println("search: "+jo);
if(jo.get("count")==null) jo.addProperty("count",10);
if(jo.get("offset")==null) jo.addProperty("offset",0);
return metaIndex.search(jo);
}
return "failed";
}
//根据时间范围查询 from to
public String searchByRange(ScriptObjectMirror so){
if(JSONTool.convertMirrorToJson(so).isJsonObject()){
JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject();
if(jo.get("count")==null) jo.addProperty("count",10);
if(jo.get("offset")==null) jo.addProperty("offset",0);
return metaIndex.searchByRange(jo);
}
return "failed";
}
public String search(String type,String keyword,int count,int offset){
JsonObject jo =new JsonObject();
jo.addProperty("type",type);
jo.addProperty("keyword",keyword);
jo.addProperty("offset",offset);
jo.addProperty("count",count);
return metaIndex.search(jo);
}
public String search(String type,String keyword,String count,String offset){
JsonObject jo =new JsonObject();
jo.addProperty("type",type);
jo.addProperty("keyword",keyword);
jo.addProperty("offset",offset);
jo.addProperty("count",count);
return metaIndex.search(jo);
}
/*
{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0}
*/
public String searchByMultiTag(ScriptObjectMirror so){
JsonElement je=JSONTool.convertMirrorToJson(so);
JsonObject jo =je.getAsJsonObject();
return metaIndex.searchByMultiTag(jo);
}
//{"doid":"xxx"}
public String delete(ScriptObjectMirror so){
if(JSONTool.convertMirrorToJson(so).isJsonObject()){
JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject();
if(jo.get("doId")==null) return "failed";
else{
metaIndex.deleteRepo(jo.get("doId").getAsString());
return "success";
}
}
return "failed";
}
public String delete(String doId){
metaIndex.deleteRepo(doId);
return "success";
}
public String search(String type, String keyword){
JsonObject jo =new JsonObject();
jo.addProperty("type",type);
jo.addProperty("keyword",keyword);
jo.addProperty("offset",0);
jo.addProperty("count",10);
return metaIndex.search(jo);
}
public int count(){
return 0;
}
//一个do或者一个do数组
public String update(ScriptObjectMirror so) throws IOException, ParseException {
JsonElement je=JSONTool.convertMirrorToJson(so);
// if(je.isJsonArray()){
// List<DoMeta> list=new ArrayList<>();
// for(JsonElement s : je.getAsJsonArray()) {
// DoMeta r = new DoMeta(s.getAsJsonObject());
// list.add(r);
// }
// metaIndex.updateRepoList(list);
// return "success";
// }
if(je.isJsonObject()){
DoMeta r=new DoMeta(je.getAsJsonObject());
metaIndex.updateRepo(r);
return "success";
}
else return "failed";
}
public String testScriptObjectMirror(ScriptObjectMirror so) throws IOException, ParseException {
//JsonObject jo = (JsonObject) ScriptUtils.convert(so, JsonObject.class);
JsonElement je=JSONTool.convertMirrorToJson(so);
System.out.println(so.toString());
System.out.println(so.getClass());
System.out.println(je.getClass());
System.out.println(je.toString());
return "success";
}
public void addFunction(String function){
try{
ContractProcess.instance.engine.compileFunction(function,false);
//ContractProcess.instance.engine.invokeFunction(function,args);
//ContractProcess.instance.engine
}catch (ScriptException e){
e.printStackTrace();
}
}
// public String updateList(JsonArray ja) throws IOException, ParseException {
// List<RepoMeta> list=new ArrayList<>();
// for(JsonElement s : ja.getAsJsonArray()){
// RepoMeta r=new RepoMeta(s.getAsJsonObject());
// list.add(r);
// }
// metaIndex.updateRepoList(list);
// return "success";
// }
}

View File

@@ -0,0 +1,5 @@
public class DebugMain {
public static void main(String[]args){
org.bdware.sc.debugger.DebugMain.main(null);
}
}

View File

@@ -0,0 +1,54 @@
import com.google.gson.JsonObject;
import org.apache.log4j.Logger;
import org.bdware.doip.audit.EndpointConfig;
import org.bdware.doip.audit.client.AuditIrpClient;
import org.bdware.doip.audit.config.FileStorage;
import org.bdware.irp.exception.IrpClientException;
import org.bdware.irp.stateinfo.StateInfoBase;
import org.junit.Before;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
public class GenerateDO {
static Logger LOGGER = Logger.getLogger(GenerateDO.class);
AuditIrpClient client;
JsonObject otherConfig;
@Before
public void init() throws FileNotFoundException {
FileStorage fs = new FileStorage("./input/auditrepo.json");
EndpointConfig config = fs.loadAsEndpointConfig();
config.repoName = null;
otherConfig = fs.load();
client = new AuditIrpClient(config);
}
@Test
public void test() throws Exception {
FileOutputStream fout = new FileOutputStream("./input/doi.txt",true);
PrintStream ps = new PrintStream(fout);
StateInfoBase base = new StateInfoBase();
base.handleValues = new JsonObject();
base.handleValues.addProperty("repoId", otherConfig.get("targetRepo").getAsString());
base.handleValues.addProperty("timestamp", System.currentTimeMillis());
try {
String doid = client.register(base);
if (doid != null) ps.println(doid);
String format = "http://%s/SCIDE/SCManager?action=executeContract&contractID=SearchEngine&operation=testAddDo&arg=";
String argStr = "{\"doId\":\"%s\",\"name\":\"repofile\",\"type\":\"text\",\"description\":\"这是一个模拟DO\",\"owner\":\"xx省yy市zz部门\",\"createTime\":%d}";
format += argStr;
String seURI = otherConfig.get("seURI").getAsString();
long createTime = System.currentTimeMillis();
long l = (long) (Math.random() * 1000D * 3600D * 24D);
String data = String.format(format, seURI, doid, createTime - l);
System.out.println(data);
} catch (IrpClientException e) {
e.printStackTrace();
}
fout.close();
ps.close();
}
}

View File

@@ -0,0 +1,26 @@
import com.google.gson.JsonObject;
import org.bdware.datanet.searchengine.MetaGraph;
import org.junit.Test;
public class GraphDBTEst {
@Test
public void Test(){
MetaGraph m=new MetaGraph("test");
JsonObject do1=new JsonObject();
do1.addProperty("id", "1");
JsonObject do2=new JsonObject();
do2.addProperty("id","2");
m.createDoNode("001",do1);
m.createDoNode("002",do2);
JsonObject jo=new JsonObject();
jo.addProperty("id1","001");
jo.addProperty("id2","002");
jo.addProperty("key1","stu_no");
jo.addProperty("key2","student_no");
m.createRelationDo2Do(jo);
m.iterDO();
m.iterRealField();
m.iterStandardField();
}
}

View File

@@ -0,0 +1,69 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.junit.Test;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
//import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.StringReader;
public class luceneTest {
@Test
public void StandardAnalyzerTest() throws Exception{
//英文测试
String text="An IndexWriter creaters and maintains an index.";
Analyzer analyzer=new StandardAnalyzer();
displayTokens(analyzer,text);
//中文测试
String text1="我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。";
displayTokens(analyzer,text1);
}
/**
58 * IKAnalyzerTest分词法测试,对中文支持很好,词库分词
59 * @throws Exception
60 */
@Test
public void IKAnalyzerTest() throws Exception{
//英文测试
String text="An IndexWriter creaters and maintains an index.";
// Analyzer analyzer=new IKAnalyzer();
Analyzer analyzer=new StandardAnalyzer();
displayTokens(analyzer,text);
//中文测试
String text1="我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。";
displayTokens(analyzer,text1);
}
public static void displayTokens(Analyzer analyzer, String text) throws Exception {
System.out.println("当前使用的分词器:" + analyzer.getClass().getName());
//分词流即将对象分词后所得的Token在内存中以流的方式存在也说是说如果在取得Token必须从TokenStream中获取而分词对象可以是文档文本也可以是查询文本。
TokenStream tokenStream = analyzer.tokenStream("content", new StringReader(text));
//表示token的首字母和尾字母在原文本中的位置。比如I'm的位置信息就是(0,3)需要注意的是startOffset与endOffset的差值并不一定就是termText.length()
//因为可能term已经用stemmer或者其他过滤器处理过
OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class);
//这个有点特殊它表示tokenStream中的当前token与前一个token在实际的原文本中相隔的词语数量用于短语查询。比如 在tokenStream中[2:a]的前一个token是[1:I'm ]
//它们在原文本中相隔的词语数是1则token="a"的PositionIncrementAttribute值为1
PositionIncrementAttribute positionIncrementAttribute = tokenStream.addAttribute(PositionIncrementAttribute.class);
CharTermAttribute charTermAttribute= tokenStream.addAttribute(CharTermAttribute.class);
//表示token词典类别信息默认为“Word”比如I'm就属于<APOSTROPHE>,有撇号的类型;
TypeAttribute typeAttribute = tokenStream.addAttribute(TypeAttribute.class);
tokenStream.reset();
int position = 0;
while (tokenStream.incrementToken()) {
int increment = positionIncrementAttribute.getPositionIncrement();
if(increment > 0) {
position = position + increment;
}
int startOffset = offsetAttribute.startOffset();
int endOffset = offsetAttribute.endOffset();
String term ="输出结果为:"+ charTermAttribute.toString();
System.out.println(""+position+"个分词,分词内容是:[" + term + "]" + ",分词内容的开始结束位置为:(" + startOffset + "-->" + endOffset + "),类型是:" + typeAttribute.type());
}
tokenStream.close();
}
}

View File

@@ -0,0 +1,184 @@
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import wrp.jdk.nashorn.api.scripting.NashornScriptEngine;
import wrp.jdk.nashorn.api.scripting.ScriptUtils;
import org.apache.lucene.queryparser.classic.ParseException;
import org.bdware.sc.boundry.JavaScriptEntry;
import org.bdware.sc.engine.JSONTool;
import org.junit.Test;
import org.bdware.datanet.searchengine.SearchEngine;
import wrp.jdk.nashorn.api.scripting.NashornScriptEngine;
import wrp.jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror;
import wrp.jdk.nashorn.internal.objects.Global;
import wrp.jdk.nashorn.internal.runtime.*;
import java.io.IOException;
import java.util.ArrayList;
public class searchEnginetest {
public static String dbname="MetaIndex01";
@Test
public void test2() throws IOException, ParseException{
System.out.println("hello");
//1642696178922
System.out.println(System.currentTimeMillis());
}
@Test
public void addDo() {
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
//SearchEngine se=new SearchEngine();
try{
engine.eval("var r1 = {\"url\":\"39.106.6.6:3306\",\"user\":\"loushuai\",\"password\":\"loushuai\",\"dbname\":\"newele\",\"tablename\":\"data\",\"createTime\":\"1641363811328\",\"owner\":\"zzz\",\"description\":\"这是一段说明\",\"schema\":\"{}\",\"prefix\":\"aibd.govdata.tj\",\"type\":\"SQL\",\"doId\":\"aibd.govdata.tj/DBRepo/do.newele.data\"};");
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r1"));
System.out.println(se.update((ScriptObjectMirror) engine.eval("r1")));
//System.out.println(se.search("info","产品"));
}catch(Exception e){
e.printStackTrace();
}
}
@Test
public void addDo2() {
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
//SearchEngine se=new SearchEngine();
try{
engine.eval("var r1 = {doId:\"AIBD/doip.Repo10101\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。\"};");
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r1"));
System.out.println(se.update((ScriptObjectMirror) engine.eval("r1")));
engine.eval("var r2 = {doId:\"AIBD/doip.Repo10102\", type:\"readme\", createTime:\"1842696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。顶层设计是聚焦用户感和赛道,通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能,体验度量作为闭环的评判标准。亮点是载体,优势是链路。思考整个生命周期,完善逻辑,考虑资源倾斜。方法论是组合拳达到平台化标准。\"};");
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r2"));
System.out.println(se.update((ScriptObjectMirror) engine.eval("r2")));
//System.out.println(se.search("info","产品"));
}catch(Exception e){
e.printStackTrace();
}
}
@Test
public void addDoList()
{
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
try{
engine.eval("var j1 = [{doId:\"AIBD/doip.Repo10101\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障……\"}," +
"{doId:\"AIBD/doip.Repo10102\", type:\"readme\", createTime:\"1642696198922\",info:\"你们这款产品底层逻辑是打通信息屏障……\"}];");
se.update((ScriptObjectMirror) engine.eval("j1"));
System.out.println(se.search("info","产品"));
}catch(Exception e){
e.printStackTrace();
}
}
@Test
public void searchRepoTest() throws IOException, ParseException {
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
try{
engine.eval("var arg ={type:\"doId\",keyword:\"AIBD/doip.Repo\"};");
System.out.println(se.search((ScriptObjectMirror) engine.eval("arg")));
engine.eval("var arg1 ={type:\"owner\",keyword:\"zzz\"};");
System.out.println(se.search((ScriptObjectMirror) engine.eval("arg1")));
//System.out.println(se.search("info","产品",5,0));
}catch(Exception e){
e.printStackTrace();
}
}
@Test
public void searchByRangeTest() throws IOException, ParseException {
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
try{
engine.eval("var arg ={type:\"createTime\",from:\"1642696178922\",to:\"1942696178922\"};");
//engine.eval("var arg ={type:\"createTime\",keyword:\"createTime\",from:\"1642696178922\",to:\"1642696178922\"};");
System.out.println(se.searchByRange ((ScriptObjectMirror) engine.eval("arg")));
}catch(Exception e){
e.printStackTrace();
}
}
//{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0}
@Test
public void searchByMultiTagTest() {
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
try{
// engine.eval("var r1 = {doId:\"repo1\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。\"};");
// System.out.println(se.update((ScriptObjectMirror) engine.eval("r1")));
//
// engine.eval("var r2 = {doId:\"repo2\", type:\"sql\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障,创建行业生态。\"};");
// System.out.println(se.update((ScriptObjectMirror) engine.eval("r2")));
engine.eval("var arg2 ={type:\"info\",keyword:\"我们\",count:10,offset:0};");
System.out.println(se.search ((ScriptObjectMirror) engine.eval("arg2")));
engine.eval("var arg1 ={params:[{type:\"doId\",keyword:\"AIBDdoip.Repo1001\"}],count:10,offset:0};");
System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg1")));
engine.eval("var arg ={params:[{type:\"info\",keyword:\"我们\"},{type:\"doId\",keyword:\"repo\"}],count:10,offset:0};");
System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg")));
engine.eval("var arg ={\n" +
" params: [{ type: 'doId', keyword: 'd' }],\n" +
" count: 100,\n" +
" offset: 0,\n" +
" };");
System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg")));
}catch(Exception e){
e.printStackTrace();
}
}
@Test
public void deleteTest(){
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
try{
engine.eval("var r1 = {doId:\"AIBD/doip.Repo1001\"};");
System.out.println(se.delete((ScriptObjectMirror)engine.eval("r1")));
//se.delele((ScriptObjectMirror)engine.eval("doid"));
System.out.println(se.search("info","产品"));
System.out.println(se.search("doId","AIBD/doip.Repo1002"));
}catch(Exception e){
e.printStackTrace();
}
}
@Test
public void jsTest(){
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6");
SearchEngine se=SearchEngine.createObj(dbname);
try{
System.out.println(engine.eval("var car = {name:\"Fiat\", model:500, color:\"white\"};"));
System.out.println(engine.eval("car").getClass());
se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("car"));
}catch(Exception e){
e.printStackTrace();
}
}
}