From 98f3866f5f9d645717620a094ba8bb99cb9834bd Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Wed, 4 Oct 2017 14:39:48 +0200 Subject: [PATCH 01/16] No more schema npe's --- .../implementations/json/JsonSchemaStore.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/implementations/json/JsonSchemaStore.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/implementations/json/JsonSchemaStore.java index 5e27438a83..741a109500 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/implementations/json/JsonSchemaStore.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/implementations/json/JsonSchemaStore.java @@ -2,15 +2,16 @@ import com.fasterxml.jackson.core.type.TypeReference; import nl.knaw.huygens.timbuctoo.v5.dataset.DataProvider; -import nl.knaw.huygens.timbuctoo.v5.jsonfilebackeddata.JsonFileBackedData; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaEntityProcessor; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaStore; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaUpdateException; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaUpdater; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Type; +import nl.knaw.huygens.timbuctoo.v5.jsonfilebackeddata.JsonFileBackedData; import java.io.File; import java.io.IOException; +import java.util.Collections; import java.util.Map; public class JsonSchemaStore implements SchemaStore { @@ -38,7 +39,12 @@ public JsonSchemaStore(DataProvider dataProvider, File schemaLocation) @Override public Map getTypes() { - return schemaFile.getData(); + final Map data = schemaFile.getData(); + if (data == null) { + return Collections.emptyMap(); + } else { + return data; + } } @Override From adb657ab24be1fc022565f1e97aaf24460075f78 Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Wed, 4 Oct 2017 14:39:57 +0200 Subject: [PATCH 02/16] show the selected mediatype --- .../src/main/resources/static/graphiql/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/timbuctoo-instancev4/src/main/resources/static/graphiql/index.html b/timbuctoo-instancev4/src/main/resources/static/graphiql/index.html index f97cb434e6..65d69b0735 100644 --- a/timbuctoo-instancev4/src/main/resources/static/graphiql/index.html +++ b/timbuctoo-instancev4/src/main/resources/static/graphiql/index.html @@ -36,6 +36,7 @@ function setAcceptMediaType(mediaType) { acceptMediaType = mediaType; + reRender(); } function graphQLFetcher(graphQLParams) { @@ -88,7 +89,7 @@ null, React.createElement( GraphiQL.Menu, - {label: "select media type"}, + {label: acceptMediaType}, mediaTypeMenuItems ), ) From 43feeb35dba65a219f8d15212d2ec9a30188f99d Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Wed, 4 Oct 2017 20:08:31 +0200 Subject: [PATCH 03/16] don't crash on ` rdf:type "something"` rdf: type should have an IRI as object, but of course in the real world it might not. We now ignore them and set the type as tim:UNKNOWN if no other types are found --- .../v5/datastores/schemastore/SchemaEntityProcessor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/schemastore/SchemaEntityProcessor.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/schemastore/SchemaEntityProcessor.java index aba14e40fb..1dbc4f5506 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/schemastore/SchemaEntityProcessor.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/schemastore/SchemaEntityProcessor.java @@ -1,10 +1,11 @@ package nl.knaw.huygens.timbuctoo.v5.datastores.schemastore; import com.google.common.collect.ListMultimap; -import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction; import nl.knaw.huygens.timbuctoo.v5.dataset.EntityProcessor; -import nl.knaw.huygens.timbuctoo.v5.dataset.dto.PredicateData; import nl.knaw.huygens.timbuctoo.v5.dataset.PredicateHandler; +import nl.knaw.huygens.timbuctoo.v5.dataset.dto.PredicateData; +import nl.knaw.huygens.timbuctoo.v5.dataset.dto.RelationPredicate; +import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Predicate; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Type; import nl.knaw.huygens.timbuctoo.v5.util.RdfConstants; @@ -45,7 +46,7 @@ public void processEntity(String cursor, String subjectUri, ListMultimap curTypes = new HashMap<>(); List subjectTypes = addedPredicates.get(RDF_TYPE); - if (subjectTypes.isEmpty()) { + if (subjectTypes.stream().noneMatch(p -> p instanceof RelationPredicate)) { curTypes.put(UNKNOWN, types.computeIfAbsent(UNKNOWN, TYPE_MAKER)); } else { for (PredicateData type : subjectTypes) { From 7a3740179daf7a2593f7f7447e2417b22b1bce82 Mon Sep 17 00:00:00 2001 From: Henk van den Berg Date: Tue, 26 Sep 2017 13:10:37 +0200 Subject: [PATCH 04/16] Search elastic - first test with rest client --- timbuctoo-instancev4/pom.xml | 8 +++ .../search/elastic/ElasticSearch.java | 60 ++++++++++++++++++ .../search/elastic/ElasticSearchTest.java | 62 +++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java create mode 100644 timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java diff --git a/timbuctoo-instancev4/pom.xml b/timbuctoo-instancev4/pom.xml index 1313dabfa8..8f63ae03d0 100644 --- a/timbuctoo-instancev4/pom.xml +++ b/timbuctoo-instancev4/pom.xml @@ -302,6 +302,14 @@ 2.1.6 + + + + org.elasticsearch.client + rest + 5.5.3 + + org.jsoup diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java new file mode 100644 index 0000000000..17a608be28 --- /dev/null +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java @@ -0,0 +1,60 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.entity.NStringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +/** + * Created on 2017-09-25 15:28. + */ +public class ElasticSearch { + + public static final String METHOD_GET = "GET"; + + private final RestClient restClient; + + public ElasticSearch(String hostname, int port, String username, String password) { + Header[] headers = { + new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), + new BasicHeader("Role", "Read")}; + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + restClient = RestClient.builder(new HttpHost(hostname, port)) + .setDefaultHeaders(headers) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder builder) { + + return builder.setDefaultCredentialsProvider(credentialsProvider); + } + }) + .build(); + } + + public void query(String index, String elasticsearchQuery, Optional token, int preferredPageSize) + throws IOException { + String endpoint = index.endsWith("_search") ? index : index.endsWith("/") ? index + "_search" : index + "/_search"; + Map params = Collections.singletonMap("pretty", "true"); + HttpEntity entity = new NStringEntity(elasticsearchQuery, ContentType.APPLICATION_JSON); + Response response = restClient.performRequest(METHOD_GET, endpoint, params, entity); + System.out.println(EntityUtils.toString(response.getEntity())); + } +} diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java new file mode 100644 index 0000000000..66cb759dad --- /dev/null +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java @@ -0,0 +1,62 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.Socket; +import java.util.Optional; + +import static org.junit.Assume.assumeTrue; + +/** + * Tests in this class assume an elasticSearch instance at localhost:9200. + * The instance should have an index with the name 'bank'. + * See example at + * https://www.elastic.co/guide/en/elasticsearch/reference/current/_exploring_your_data.html#_loading_the_sample_dataset + */ +public class ElasticSearchTest { + + private static final String hostname = "localhost"; + private static final int port = 9200; + private static final String username = "elastic"; + private static final String password = "changeme"; + + private static ElasticSearch eSearch; + + @BeforeClass + public static void initialize() throws Exception { + assumeTrue("No host at " + hostname + ":" + port + ", skipping tests", hostIsAvailable()); + eSearch = new ElasticSearch(hostname, port, username, password); + } + + private static boolean hostIsAvailable() { + try (Socket s = new Socket(hostname, port)) { + return true; + } catch (IOException ex) { + /* ignore */ + } + return false; + } + + @Test + public void querySimple() throws Exception { + String index = "bank"; + String elasticsearchQuery = "{\n" + + " \"size\": 3,\n" + + " \"query\": {\n" + + " \"match\" : {\n" + + " \"gender\" : \"F\"\n" + + " }\n" + + " },\n" + + " \"sort\": [\n" + + " {\"balance\": \"asc\"},\n" + + " {\"account_number\": \"desc\"}\n" + + " ]\n" + + "}"; + Optional token = Optional.empty(); + int preferredPageSize = 3; + eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + } + +} From f6efc3cb1321a9693fa913525338ba23ef628d12 Mon Sep 17 00:00:00 2001 From: Henk van den Berg Date: Mon, 2 Oct 2017 12:49:35 +0200 Subject: [PATCH 05/16] Enhance query as JsonNode --- .../search/elastic/ElasticSearch.java | 46 +++++--- .../search/elastic/PageableResult2.java | 39 +++++++ .../elastic/ElasticSearchOnLineTest.java | 106 ++++++++++++++++++ .../search/elastic/ElasticSearchTest.java | 62 ---------- 4 files changed, 176 insertions(+), 77 deletions(-) create mode 100644 timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java create mode 100644 timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java delete mode 100644 timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java index 17a608be28..71d6fab164 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java @@ -1,5 +1,7 @@ package nl.knaw.huygens.timbuctoo.search.elastic; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; @@ -9,28 +11,26 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.entity.ContentType; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHeader; import org.apache.http.nio.entity.NStringEntity; -import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.Collections; +import java.util.List; import java.util.Map; -import java.util.Optional; /** * Created on 2017-09-25 15:28. */ public class ElasticSearch { - public static final String METHOD_GET = "GET"; + public static final String FIELD_NAME = "_uid"; + private static final String METHOD_GET = "GET"; private final RestClient restClient; + private final ObjectMapper mapper; public ElasticSearch(String hostname, int port, String username, String password) { Header[] headers = { @@ -40,21 +40,37 @@ public ElasticSearch(String hostname, int port, String username, String password credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); restClient = RestClient.builder(new HttpHost(hostname, port)) .setDefaultHeaders(headers) - .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { - public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder builder) { - - return builder.setDefaultCredentialsProvider(credentialsProvider); - } - }) + .setHttpClientConfigCallback( + builder -> builder.setDefaultCredentialsProvider(credentialsProvider)) .build(); + mapper = new ObjectMapper(); } - public void query(String index, String elasticsearchQuery, Optional token, int preferredPageSize) + public PageableResult2 query(String index, String elasticSearchQuery, String token, int preferredPageSize) throws IOException { String endpoint = index.endsWith("_search") ? index : index.endsWith("/") ? index + "_search" : index + "/_search"; Map params = Collections.singletonMap("pretty", "true"); - HttpEntity entity = new NStringEntity(elasticsearchQuery, ContentType.APPLICATION_JSON); + HttpEntity entity = new NStringEntity(elasticSearchQuery, ContentType.APPLICATION_JSON); Response response = restClient.performRequest(METHOD_GET, endpoint, params, entity); - System.out.println(EntityUtils.toString(response.getEntity())); + + // System.out.println(EntityUtils.toString(response.getEntity())); + JsonNode jsonNode = mapper.readTree(response.getEntity().getContent()); + System.err.println("++" + "\n" + jsonNode.toString()); + List sortNodes = jsonNode.findValues("sort"); + for (JsonNode node : sortNodes) { + System.out.println(node.toString()); + } + // for (JsonNode node : sortNodes) { + // System.out.println(node.getNodeType().name()); + // } + return new PageableResult2().setIdFields(jsonNode.findValuesAsText(FIELD_NAME)); } + + String elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) throws IOException { + JsonNode query = mapper.readTree(elasticSearchQuery); + + return null; + } + + } diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java new file mode 100644 index 0000000000..1a489c61b8 --- /dev/null +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java @@ -0,0 +1,39 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Created on 2017-09-28 12:56. + */ +public class PageableResult2 { + + private List idFields = new ArrayList<>(); + private String token; + + public List getIdFields() { + return idFields; + } + + PageableResult2 setIdFields(List idFields) { + this.idFields = idFields; + return this; + } + + public Optional getToken() { + if (token == null) { + return Optional.empty(); + } else { + return Optional.of(token); + } + } + + PageableResult2 setToken(String token) { + this.token = token; + return this; + } + + +} diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java new file mode 100644 index 0000000000..d126d7634e --- /dev/null +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java @@ -0,0 +1,106 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.Socket; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assume.assumeTrue; + +/** + * Tests in this class assume an elasticSearch instance at localhost:9200. + * The instance should have an index with the name 'bank'. + * See example at + * https://www.elastic.co/guide/en/elasticsearch/reference/current/_exploring_your_data.html#_loading_the_sample_dataset + */ +public class ElasticSearchOnLineTest { + + private static final String hostname = "localhost"; + private static final int port = 9200; + private static final String username = "elastic"; + private static final String password = "changeme"; + + private static ElasticSearch eSearch; + + @BeforeClass + public static void initialize() throws Exception { + assumeTrue("No host at " + hostname + ":" + port + ", skipping tests." + + "\nPlease start an ElasticSearch instance at the specified host and port.", hostIsAvailable()); + eSearch = new ElasticSearch(hostname, port, username, password); + } + + private static boolean hostIsAvailable() { + try (Socket s = new Socket(hostname, port)) { + return true; + } catch (IOException ex) { + /* ignore */ + } + return false; + } + + @Test + public void querySimple() throws Exception { + String index = "bank"; + String elasticsearchQuery = createQuery1(); + String token = null; + int preferredPageSize = 3; + PageableResult2 result = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + + //assertThat(result.getIdFields().size(), equalTo(3)); + //assertThat(result.getIdFields(), contains("820", "315", "490")); + + } + + @Test(expected = org.elasticsearch.client.ResponseException.class) + public void queryAndIndexDoesNotExist() throws Exception { + String index = "does_not_exist"; + String elasticsearchQuery = createQuery1(); + String token = null; + int preferredPageSize = 3; + eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + } + + @Test + public void queryWithFieldName() throws Exception { + String index = "bank"; + String elasticsearchQuery = createQuery2(); + String token = null; + int preferredPageSize = 3; + PageableResult2 result = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + System.out.println(result.getIdFields()); + } + + private String createQuery1() { + return "{\n" + + " \"size\": 3,\n" + + " \"query\": {\n" + + " \"match\" : {\n" + + " \"gender\" : \"F\"\n" + + " }\n" + + " },\n" + + " \"sort\": [\n" + + " {\"balance\": \"asc\"},\n" + + " {\"account_number\": \"desc\"}\n" + + " ]\n" + + "}"; + } + + private String createQuery2() { + return "{\n" + + " \"size\": 3,\n" + + " \"query\": {\n" + + " \"match\" : {\n" + + " \"gender\" : \"F\"\n" + + " }\n" + + " },\n" + + " \"sort\": [\n" + + " {\"balance\": \"asc\"},\n" + + " {\"" + ElasticSearch.FIELD_NAME + "\": \"desc\"}\n" + + " ]\n" + + "}"; + } +} diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java deleted file mode 100644 index 66cb759dad..0000000000 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; - -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.net.Socket; -import java.util.Optional; - -import static org.junit.Assume.assumeTrue; - -/** - * Tests in this class assume an elasticSearch instance at localhost:9200. - * The instance should have an index with the name 'bank'. - * See example at - * https://www.elastic.co/guide/en/elasticsearch/reference/current/_exploring_your_data.html#_loading_the_sample_dataset - */ -public class ElasticSearchTest { - - private static final String hostname = "localhost"; - private static final int port = 9200; - private static final String username = "elastic"; - private static final String password = "changeme"; - - private static ElasticSearch eSearch; - - @BeforeClass - public static void initialize() throws Exception { - assumeTrue("No host at " + hostname + ":" + port + ", skipping tests", hostIsAvailable()); - eSearch = new ElasticSearch(hostname, port, username, password); - } - - private static boolean hostIsAvailable() { - try (Socket s = new Socket(hostname, port)) { - return true; - } catch (IOException ex) { - /* ignore */ - } - return false; - } - - @Test - public void querySimple() throws Exception { - String index = "bank"; - String elasticsearchQuery = "{\n" + - " \"size\": 3,\n" + - " \"query\": {\n" + - " \"match\" : {\n" + - " \"gender\" : \"F\"\n" + - " }\n" + - " },\n" + - " \"sort\": [\n" + - " {\"balance\": \"asc\"},\n" + - " {\"account_number\": \"desc\"}\n" + - " ]\n" + - "}"; - Optional token = Optional.empty(); - int preferredPageSize = 3; - eSearch.query(index, elasticsearchQuery, token, preferredPageSize); - } - -} From 05cedf1a06e9a211942c89747028ab77a988b736 Mon Sep 17 00:00:00 2001 From: Henk van den Berg Date: Mon, 2 Oct 2017 14:09:29 +0200 Subject: [PATCH 06/16] Elaborate query --- .../search/elastic/ElasticSearch.java | 11 ++- .../search/elastic/ElasticSearchTest.java | 79 +++++++++++++++++++ 2 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java index 71d6fab164..091484ad4b 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; @@ -66,10 +67,12 @@ public PageableResult2 query(String index, String elasticSearchQuery, String tok return new PageableResult2().setIdFields(jsonNode.findValuesAsText(FIELD_NAME)); } - String elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) throws IOException { - JsonNode query = mapper.readTree(elasticSearchQuery); - - return null; + ObjectNode elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) throws IOException { + ObjectNode qNode = (ObjectNode) mapper.readTree(elasticSearchQuery); + if (preferredPageSize > 0) { + qNode.put("size", preferredPageSize); + } + return qNode; } diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java new file mode 100644 index 0000000000..4873b2bc22 --- /dev/null +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java @@ -0,0 +1,79 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assume.assumeTrue; + +/** + * Created on 2017-10-02 12:57. + */ +public class ElasticSearchTest { + + private static ElasticSearch eSearch; + + @BeforeClass + public static void initialize() throws Exception { + eSearch = new ElasticSearch("", 0, "", null); + } + + @Test + public void elaborateQueryAsIs() throws Exception { + String originalQuery = createQuery1(); + JsonNode qAsNode = eSearch.elaborateQuery(originalQuery, null, -1); + + assertThat(qAsNode.toString(), equalTo(originalQuery.replaceAll("\n|\\s", ""))); + JsonNode sizeNode = qAsNode.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(3)); + } + + @Test + public void elaborateQueryWithSize() throws Exception { + String originalQuery = createQuery1(); + JsonNode qAsNode = eSearch.elaborateQuery(originalQuery, null, 4); + + JsonNode sizeNode = qAsNode.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(4)); + } + + @Test + public void elaborateQueryWithoutSize() throws Exception { + String originalQuery = createQueryWithoutSize(); + JsonNode qAsNode = eSearch.elaborateQuery(originalQuery, null, 4); + + JsonNode sizeNode = qAsNode.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(4)); + } + + private String createQuery1() { + return "{\n" + + " \"size\": 3,\n" + + " \"query\": {\n" + + " \"match\" : {\n" + + " \"gender\" : \"F\"\n" + + " }\n" + + " },\n" + + " \"sort\": [\n" + + " {\"balance\": \"asc\"},\n" + + " {\"account_number\": \"desc\"}\n" + + " ]\n" + + "}"; + } + + private String createQueryWithoutSize() { + return "{\n" + + " \"query\": {\n" + + " \"match\" : {\n" + + " \"gender\" : \"F\"\n" + + " }\n" + + " },\n" + + " \"sort\": [\n" + + " {\"balance\": \"asc\"},\n" + + " {\"account_number\": \"desc\"}\n" + + " ]\n" + + "}"; + } +} From 78f27264e75dcf0459f820fc66f640b2a96133bc Mon Sep 17 00:00:00 2001 From: Henk van den Berg Date: Mon, 2 Oct 2017 14:21:51 +0200 Subject: [PATCH 07/16] .. variable names !like qNode .. --- .../timbuctoo/search/elastic/ElasticSearch.java | 6 +++--- .../search/elastic/ElasticSearchTest.java | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java index 091484ad4b..99d0293f56 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java @@ -68,11 +68,11 @@ public PageableResult2 query(String index, String elasticSearchQuery, String tok } ObjectNode elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) throws IOException { - ObjectNode qNode = (ObjectNode) mapper.readTree(elasticSearchQuery); + ObjectNode node = (ObjectNode) mapper.readTree(elasticSearchQuery); if (preferredPageSize > 0) { - qNode.put("size", preferredPageSize); + node.put("size", preferredPageSize); } - return qNode; + return node; } diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java index 4873b2bc22..62a0e83afc 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java @@ -23,28 +23,28 @@ public static void initialize() throws Exception { @Test public void elaborateQueryAsIs() throws Exception { String originalQuery = createQuery1(); - JsonNode qAsNode = eSearch.elaborateQuery(originalQuery, null, -1); + JsonNode queryAsNode = eSearch.elaborateQuery(originalQuery, null, -1); - assertThat(qAsNode.toString(), equalTo(originalQuery.replaceAll("\n|\\s", ""))); - JsonNode sizeNode = qAsNode.findValue("size"); + assertThat(queryAsNode.toString(), equalTo(originalQuery.replaceAll("\n|\\s", ""))); + JsonNode sizeNode = queryAsNode.findValue("size"); assertThat(sizeNode.asInt(), equalTo(3)); } @Test public void elaborateQueryWithSize() throws Exception { String originalQuery = createQuery1(); - JsonNode qAsNode = eSearch.elaborateQuery(originalQuery, null, 4); + JsonNode queryAsNode = eSearch.elaborateQuery(originalQuery, null, 4); - JsonNode sizeNode = qAsNode.findValue("size"); + JsonNode sizeNode = queryAsNode.findValue("size"); assertThat(sizeNode.asInt(), equalTo(4)); } @Test public void elaborateQueryWithoutSize() throws Exception { String originalQuery = createQueryWithoutSize(); - JsonNode qAsNode = eSearch.elaborateQuery(originalQuery, null, 4); + JsonNode queryAsNode = eSearch.elaborateQuery(originalQuery, null, 4); - JsonNode sizeNode = qAsNode.findValue("size"); + JsonNode sizeNode = queryAsNode.findValue("size"); assertThat(sizeNode.asInt(), equalTo(4)); } From 5f4ef570537c4adcf0e2f805ec0aacf66f19154c Mon Sep 17 00:00:00 2001 From: Henk van den Berg Date: Tue, 3 Oct 2017 14:29:51 +0200 Subject: [PATCH 08/16] Elaborate query complete --- .../search/elastic/ElasticSearch.java | 30 +++++- .../search/elastic/ElasticSearchTest.java | 99 +++++++++++++++---- 2 files changed, 106 insertions(+), 23 deletions(-) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java index 99d0293f56..bfd38d4de8 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -68,9 +70,33 @@ public PageableResult2 query(String index, String elasticSearchQuery, String tok } ObjectNode elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) throws IOException { + // size -1 gives the default 10 results. size 0 gives 0 results. totals are always given. + // requests without a 'query' clause are legal, so don't check. + // if 'search_after' is present, 'sort' must contain just as many fields of same type (not checked). + // if 'search_after' is present, 'sort' must contain "..one unique value per document.." + // (we check on/put FIELD_NAME). ObjectNode node = (ObjectNode) mapper.readTree(elasticSearchQuery); - if (preferredPageSize > 0) { - node.put("size", preferredPageSize); + node.put("size", preferredPageSize); + if (token != null) { + if (!node.has("search_after")) { + node.putArray("search_after"); + } + ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); + searchAfterNode.removeAll(); + searchAfterNode.addAll((ArrayNode) mapper.readTree(token)); + + if (!node.has("sort")) { + node.putArray("sort"); + } + ArrayNode sortNode = (ArrayNode) node.findValue("sort"); + if (sortNode.findValue(FIELD_NAME) == null) { + ObjectNode objNode = JsonNodeFactory.instance.objectNode(); + objNode.put(FIELD_NAME, "desc"); + sortNode.add(objNode); + } + + } else { + node.remove("search_after"); } return node; } diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java index 62a0e83afc..e880dc8433 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java @@ -1,9 +1,14 @@ package nl.knaw.huygens.timbuctoo.search.elastic; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.junit.BeforeClass; import org.junit.Test; +import java.util.Iterator; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assume.assumeTrue; @@ -21,34 +26,83 @@ public static void initialize() throws Exception { } @Test - public void elaborateQueryAsIs() throws Exception { - String originalQuery = createQuery1(); - JsonNode queryAsNode = eSearch.elaborateQuery(originalQuery, null, -1); + public void elaborateEmptyQuery() throws Exception { + JsonNode node = eSearch.elaborateQuery(createEmptyQuery(), null, 5); + JsonNode sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(5)); + assertThat(node.has("search_after"), equalTo(false)); + //System.out.println(node.toString()); // {"size":5} + + ArrayNode token = JsonNodeFactory.instance.arrayNode(); + token.add(123); - assertThat(queryAsNode.toString(), equalTo(originalQuery.replaceAll("\n|\\s", ""))); - JsonNode sizeNode = queryAsNode.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(3)); + node = eSearch.elaborateQuery(createEmptyQuery(), token.toString(), 6); + sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(6)); + ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); + assertThat(searchAfterNode.elements().next().asInt(), equalTo(123)); + ArrayNode sortNode = (ArrayNode) node.findValue("sort"); + assertThat(sortNode.elements().next().toString(), equalTo("{\"_uid\":\"desc\"}")); + //System.out.println(node.toString()); // {"size":6,"search_after":[123],"sort":[{"_uid":"desc"}]} } @Test - public void elaborateQueryWithSize() throws Exception { - String originalQuery = createQuery1(); - JsonNode queryAsNode = eSearch.elaborateQuery(originalQuery, null, 4); + public void elaborateCompleteQuery() throws Exception { + JsonNode node = eSearch.elaborateQuery(createCompleteQuery(), null, 5); + JsonNode sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(5)); + assertThat(node.has("search_after"), equalTo(false)); + ArrayNode sortNode = (ArrayNode) node.findValue("sort"); + assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]")); + //System.out.println(node.toString()); + // {"size":5,"query":{"match":{"gender":"F"}},"sort":[{"balance":"asc"},{"_uid":"desc"}]} + + ArrayNode token = JsonNodeFactory.instance.arrayNode(); + token.add(49223); + token.add(123); - JsonNode sizeNode = queryAsNode.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(4)); + node = eSearch.elaborateQuery(createCompleteQuery(), token.toString(), 6); + sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(6)); + ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); + assertThat(searchAfterNode.toString(), equalTo("[49223,123]")); + sortNode = (ArrayNode) node.findValue("sort"); + assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]")); + //System.out.println(node.toString()); + // {"size":6,"query":{"match":{"gender":"F"}},"search_after":[49223,123],"sort":[{"balance":"asc"},{"_uid":"desc"}]} } @Test - public void elaborateQueryWithoutSize() throws Exception { - String originalQuery = createQueryWithoutSize(); - JsonNode queryAsNode = eSearch.elaborateQuery(originalQuery, null, 4); + public void elaborateIncompleteQuery() throws Exception { + JsonNode node = eSearch.elaborateQuery(createIncompleteQuery(), null, 5); + JsonNode sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(5)); + assertThat(node.has("search_after"), equalTo(false)); + ArrayNode sortNode = (ArrayNode) node.findValue("sort"); + assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"}]")); + //System.out.println(node.toString()); + // {"size":5,"query":{"match":{"gender":"F"}},"sort":[{"balance":"asc"}]} + + ArrayNode token = JsonNodeFactory.instance.arrayNode(); + token.add(49223); + token.add(123); + + node = eSearch.elaborateQuery(createCompleteQuery(), token.toString(), 6); + sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(6)); + ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); + assertThat(searchAfterNode.toString(), equalTo("[49223,123]")); + sortNode = (ArrayNode) node.findValue("sort"); + assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]")); + //System.out.println(node.toString()); + //{"size":6,"query":{"match":{"gender":"F"}},"search_after":[49223,123],"sort":[{"balance":"asc"},{"_uid":"desc"}]} + } - JsonNode sizeNode = queryAsNode.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(4)); + private String createEmptyQuery() { + return "{}"; } - private String createQuery1() { + private String createCompleteQuery() { return "{\n" + " \"size\": 3,\n" + " \"query\": {\n" + @@ -56,24 +110,27 @@ private String createQuery1() { " \"gender\" : \"F\"\n" + " }\n" + " },\n" + + " \"search_after\": [1314, 315],\n" + " \"sort\": [\n" + " {\"balance\": \"asc\"},\n" + - " {\"account_number\": \"desc\"}\n" + + " {\"_uid\": \"desc\"}\n" + " ]\n" + "}"; } - private String createQueryWithoutSize() { + private String createIncompleteQuery() { return "{\n" + + " \"size\": 3,\n" + " \"query\": {\n" + " \"match\" : {\n" + " \"gender\" : \"F\"\n" + " }\n" + " },\n" + " \"sort\": [\n" + - " {\"balance\": \"asc\"},\n" + - " {\"account_number\": \"desc\"}\n" + + " {\"balance\": \"asc\"}\n" + " ]\n" + "}"; } + } + From c7e21ad3b2d501d77088c4e1110ed4c612cd808d Mon Sep 17 00:00:00 2001 From: Henk van den Berg Date: Wed, 4 Oct 2017 13:21:32 +0200 Subject: [PATCH 09/16] Search ElasticSearch and get Pageable Results --- .../search/elastic/ElasticSearch.java | 61 +++--- .../search/elastic/PageableResult.java | 50 +++++ .../search/elastic/PageableResult2.java | 39 ---- .../elastic/ElasticSearchOnLineTest.java | 40 ++-- .../search/elastic/ElasticSearchTest.java | 131 +++++++------ .../search/elastic/PageableResultTest.java | 177 ++++++++++++++++++ 6 files changed, 349 insertions(+), 149 deletions(-) create mode 100644 timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java delete mode 100644 timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java create mode 100644 timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java index bfd38d4de8..f87a4b312d 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.List; import java.util.Map; /** @@ -29,7 +28,8 @@ */ public class ElasticSearch { - public static final String FIELD_NAME = "_uid"; + public static final String UNIQUE_FIELD_NAME = "_uid"; + private static final String METHOD_GET = "GET"; private final RestClient restClient; @@ -49,55 +49,52 @@ public ElasticSearch(String hostname, int port, String username, String password mapper = new ObjectMapper(); } - public PageableResult2 query(String index, String elasticSearchQuery, String token, int preferredPageSize) + public PageableResult query(String index, String elasticSearchQuery, String token, int preferredPageSize) throws IOException { String endpoint = index.endsWith("_search") ? index : index.endsWith("/") ? index + "_search" : index + "/_search"; + JsonNode queryNode = elaborateQuery(elasticSearchQuery, token, preferredPageSize); Map params = Collections.singletonMap("pretty", "true"); - HttpEntity entity = new NStringEntity(elasticSearchQuery, ContentType.APPLICATION_JSON); + + HttpEntity entity = new NStringEntity(queryNode.toString(), ContentType.APPLICATION_JSON); Response response = restClient.performRequest(METHOD_GET, endpoint, params, entity); - // System.out.println(EntityUtils.toString(response.getEntity())); - JsonNode jsonNode = mapper.readTree(response.getEntity().getContent()); - System.err.println("++" + "\n" + jsonNode.toString()); - List sortNodes = jsonNode.findValues("sort"); - for (JsonNode node : sortNodes) { - System.out.println(node.toString()); - } - // for (JsonNode node : sortNodes) { - // System.out.println(node.getNodeType().name()); - // } - return new PageableResult2().setIdFields(jsonNode.findValuesAsText(FIELD_NAME)); + JsonNode responseNode = mapper.readTree(response.getEntity().getContent()); + return new PageableResult(queryNode, responseNode); } - ObjectNode elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) throws IOException { + protected ObjectNode elaborateQuery(String elasticSearchQuery, String token, int preferredPageSize) + throws IOException { // size -1 gives the default 10 results. size 0 gives 0 results. totals are always given. // requests without a 'query' clause are legal, so don't check. // if 'search_after' is present, 'sort' must contain just as many fields of same type (not checked). - // if 'search_after' is present, 'sort' must contain "..one unique value per document.." - // (we check on/put FIELD_NAME). + // 'sort' must be present and must contain "..one unique value per document.." (we check on/put UNIQUE_FIELD_NAME). ObjectNode node = (ObjectNode) mapper.readTree(elasticSearchQuery); + + // size node.put("size", preferredPageSize); + + // search_after if (token != null) { - if (!node.has("search_after")) { - node.putArray("search_after"); - } ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); + if (searchAfterNode == null) { + searchAfterNode = node.putArray("search_after"); + } searchAfterNode.removeAll(); searchAfterNode.addAll((ArrayNode) mapper.readTree(token)); - - if (!node.has("sort")) { - node.putArray("sort"); - } - ArrayNode sortNode = (ArrayNode) node.findValue("sort"); - if (sortNode.findValue(FIELD_NAME) == null) { - ObjectNode objNode = JsonNodeFactory.instance.objectNode(); - objNode.put(FIELD_NAME, "desc"); - sortNode.add(objNode); - } - } else { node.remove("search_after"); } + + // sort + ArrayNode sortNode = (ArrayNode) node.findValue("sort"); + if (sortNode == null) { + sortNode = node.putArray("sort"); + } + if (sortNode.findValue(UNIQUE_FIELD_NAME) == null) { + ObjectNode objNode = JsonNodeFactory.instance.objectNode(); + objNode.put(UNIQUE_FIELD_NAME, "desc"); + sortNode.add(objNode); + } return node; } diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java new file mode 100644 index 0000000000..c31c02e601 --- /dev/null +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java @@ -0,0 +1,50 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.List; + +/** + * Created on 2017-10-04 10:54. + */ +public class PageableResult { + + public static final String ID_FIELD = "_id"; + + private final JsonNode queryNode; + private final JsonNode resultNode; + + public PageableResult(JsonNode queryNode, JsonNode resultNode) { + this.queryNode = queryNode; + this.resultNode = resultNode; + } + + public String getQuery() { + return queryNode.toString(); + } + + public String getResult() { + return resultNode.toString(); + } + + public List getIdList() { + return resultNode.findPath("hits").findPath("hits").findValuesAsText(ID_FIELD); + } + + public String getToken() { + String token = null; + List sortNodes = resultNode.findValues("sort"); + if (!sortNodes.isEmpty()) { + token = sortNodes.get(sortNodes.size() - 1).toString(); + } + return token; + } + + public int getTotalHits() { + return resultNode.findPath("hits").findPath("total").asInt(); + } + + public int getSearchTime() { + return resultNode.findPath("took").asInt(); + } +} diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java deleted file mode 100644 index 1a489c61b8..0000000000 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult2.java +++ /dev/null @@ -1,39 +0,0 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; - - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * Created on 2017-09-28 12:56. - */ -public class PageableResult2 { - - private List idFields = new ArrayList<>(); - private String token; - - public List getIdFields() { - return idFields; - } - - PageableResult2 setIdFields(List idFields) { - this.idFields = idFields; - return this; - } - - public Optional getToken() { - if (token == null) { - return Optional.empty(); - } else { - return Optional.of(token); - } - } - - PageableResult2 setToken(String token) { - this.token = token; - return this; - } - - -} diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java index d126d7634e..7b8a4de8cc 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java @@ -48,30 +48,47 @@ public void querySimple() throws Exception { String elasticsearchQuery = createQuery1(); String token = null; int preferredPageSize = 3; - PageableResult2 result = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + PageableResult pageableResult = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); - //assertThat(result.getIdFields().size(), equalTo(3)); - //assertThat(result.getIdFields(), contains("820", "315", "490")); + //System.out.println(pageableResult.getResult()); + token = pageableResult.getToken(); + assertThat(token, equalTo("[1447,\"account#490\"]")); + assertThat(pageableResult.getIdList(), contains("820", "315", "490")); + assertThat(pageableResult.getTotalHits(), equalTo(493)); + pageableResult = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + + //System.out.println(pageableResult.getResult()); + token = pageableResult.getToken(); + assertThat(token, equalTo("[1696,\"account#159\"]")); + assertThat(pageableResult.getIdList(), contains("427", "174", "159")); + assertThat(pageableResult.getTotalHits(), equalTo(493)); } @Test(expected = org.elasticsearch.client.ResponseException.class) public void queryAndIndexDoesNotExist() throws Exception { String index = "does_not_exist"; String elasticsearchQuery = createQuery1(); - String token = null; int preferredPageSize = 3; - eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + eSearch.query(index, elasticsearchQuery, null, preferredPageSize); } @Test - public void queryWithFieldName() throws Exception { + public void queryAndNoResult() throws Exception { String index = "bank"; String elasticsearchQuery = createQuery2(); String token = null; int preferredPageSize = 3; - PageableResult2 result = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); - System.out.println(result.getIdFields()); + PageableResult pageableResult = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + + assertThat(pageableResult.getToken(), equalTo(null)); + assertThat(pageableResult.getIdList().size(), equalTo(0)); + assertThat(pageableResult.getTotalHits(), equalTo(0)); + + token = pageableResult.getToken(); + pageableResult = eSearch.query(index, elasticsearchQuery, token, preferredPageSize); + + assertThat(pageableResult.getTotalHits(), equalTo(0)); } private String createQuery1() { @@ -83,8 +100,7 @@ private String createQuery1() { " }\n" + " },\n" + " \"sort\": [\n" + - " {\"balance\": \"asc\"},\n" + - " {\"account_number\": \"desc\"}\n" + + " {\"balance\": \"asc\"}\n" + " ]\n" + "}"; } @@ -94,12 +110,12 @@ private String createQuery2() { " \"size\": 3,\n" + " \"query\": {\n" + " \"match\" : {\n" + - " \"gender\" : \"F\"\n" + + " \"gender\" : \"U\"\n" + " }\n" + " },\n" + " \"sort\": [\n" + " {\"balance\": \"asc\"},\n" + - " {\"" + ElasticSearch.FIELD_NAME + "\": \"desc\"}\n" + + " {\"" + ElasticSearch.UNIQUE_FIELD_NAME + "\": \"desc\"}\n" + " ]\n" + "}"; } diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java index e880dc8433..bd65b2ebef 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java @@ -1,6 +1,5 @@ package nl.knaw.huygens.timbuctoo.search.elastic; -import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; @@ -11,7 +10,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assume.assumeTrue; /** * Created on 2017-10-02 12:57. @@ -27,83 +25,39 @@ public static void initialize() throws Exception { @Test public void elaborateEmptyQuery() throws Exception { - JsonNode node = eSearch.elaborateQuery(createEmptyQuery(), null, 5); + // empty query with and without token. + String query = "{}"; + String expectedSort = "{\"" + ElasticSearch.UNIQUE_FIELD_NAME + "\":\"desc\"}"; + + JsonNode node = eSearch.elaborateQuery(query, null, 5); + JsonNode sizeNode = node.findValue("size"); assertThat(sizeNode.asInt(), equalTo(5)); assertThat(node.has("search_after"), equalTo(false)); - //System.out.println(node.toString()); // {"size":5} + JsonNode sortNode = node.findValue("sort"); + assertThat(sortNode.elements().next().toString(), equalTo(expectedSort)); + String expected = "{\"size\":5,\"sort\":[" + expectedSort + "]}"; + assertThat(node.toString(), equalTo(expected)); + //System.out.println(node.toString()); // {"size":5,"sort":[{"_uid":"desc"}]} ArrayNode token = JsonNodeFactory.instance.arrayNode(); token.add(123); + expected = "{\"size\":6,\"search_after\":[123],\"sort\":[" + expectedSort + "]}"; + node = eSearch.elaborateQuery(query, token.toString(), 6); - node = eSearch.elaborateQuery(createEmptyQuery(), token.toString(), 6); sizeNode = node.findValue("size"); assertThat(sizeNode.asInt(), equalTo(6)); ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); assertThat(searchAfterNode.elements().next().asInt(), equalTo(123)); - ArrayNode sortNode = (ArrayNode) node.findValue("sort"); - assertThat(sortNode.elements().next().toString(), equalTo("{\"_uid\":\"desc\"}")); + sortNode = node.findValue("sort"); + assertThat(sortNode.elements().next().toString(), equalTo(expectedSort)); + assertThat(node.toString(), equalTo(expected)); //System.out.println(node.toString()); // {"size":6,"search_after":[123],"sort":[{"_uid":"desc"}]} } @Test public void elaborateCompleteQuery() throws Exception { - JsonNode node = eSearch.elaborateQuery(createCompleteQuery(), null, 5); - JsonNode sizeNode = node.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(5)); - assertThat(node.has("search_after"), equalTo(false)); - ArrayNode sortNode = (ArrayNode) node.findValue("sort"); - assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]")); - //System.out.println(node.toString()); - // {"size":5,"query":{"match":{"gender":"F"}},"sort":[{"balance":"asc"},{"_uid":"desc"}]} - - ArrayNode token = JsonNodeFactory.instance.arrayNode(); - token.add(49223); - token.add(123); - - node = eSearch.elaborateQuery(createCompleteQuery(), token.toString(), 6); - sizeNode = node.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(6)); - ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); - assertThat(searchAfterNode.toString(), equalTo("[49223,123]")); - sortNode = (ArrayNode) node.findValue("sort"); - assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]")); - //System.out.println(node.toString()); - // {"size":6,"query":{"match":{"gender":"F"}},"search_after":[49223,123],"sort":[{"balance":"asc"},{"_uid":"desc"}]} - } - - @Test - public void elaborateIncompleteQuery() throws Exception { - JsonNode node = eSearch.elaborateQuery(createIncompleteQuery(), null, 5); - JsonNode sizeNode = node.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(5)); - assertThat(node.has("search_after"), equalTo(false)); - ArrayNode sortNode = (ArrayNode) node.findValue("sort"); - assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"}]")); - //System.out.println(node.toString()); - // {"size":5,"query":{"match":{"gender":"F"}},"sort":[{"balance":"asc"}]} - - ArrayNode token = JsonNodeFactory.instance.arrayNode(); - token.add(49223); - token.add(123); - - node = eSearch.elaborateQuery(createCompleteQuery(), token.toString(), 6); - sizeNode = node.findValue("size"); - assertThat(sizeNode.asInt(), equalTo(6)); - ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); - assertThat(searchAfterNode.toString(), equalTo("[49223,123]")); - sortNode = (ArrayNode) node.findValue("sort"); - assertThat(sortNode.toString(), equalTo("[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]")); - //System.out.println(node.toString()); - //{"size":6,"query":{"match":{"gender":"F"}},"search_after":[49223,123],"sort":[{"balance":"asc"},{"_uid":"desc"}]} - } - - private String createEmptyQuery() { - return "{}"; - } - - private String createCompleteQuery() { - return "{\n" + + String query = "{\n" + " \"size\": 3,\n" + " \"query\": {\n" + " \"match\" : {\n" + @@ -113,23 +67,68 @@ private String createCompleteQuery() { " \"search_after\": [1314, 315],\n" + " \"sort\": [\n" + " {\"balance\": \"asc\"},\n" + - " {\"_uid\": \"desc\"}\n" + + " {\"" + ElasticSearch.UNIQUE_FIELD_NAME + "\": \"desc\"}\n" + " ]\n" + "}"; + testElaborate(query); } - private String createIncompleteQuery() { - return "{\n" + + @Test + public void elaborateIncompleteQuery() throws Exception { + // missing unique field. + String query = "{\n" + " \"size\": 3,\n" + " \"query\": {\n" + " \"match\" : {\n" + " \"gender\" : \"F\"\n" + " }\n" + " },\n" + + " \"search_after\": [1314, 315],\n" + " \"sort\": [\n" + " {\"balance\": \"asc\"}\n" + " ]\n" + "}"; + testElaborate(query); + } + + private void testElaborate(String query) throws Exception { + // test elaborate with and without token. + + JsonNode node = eSearch.elaborateQuery(query, null, 5); + + JsonNode sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(5)); + assertThat(node.has("search_after"), equalTo(false)); + JsonNode sortNode = node.findValue("sort"); + Iterator sortIter = sortNode.elements(); + String expectedSort1 = "{\"balance\":\"asc\"}"; + String expectedSort2 = "{\"" + ElasticSearch.UNIQUE_FIELD_NAME + "\":\"desc\"}"; + assertThat(sortIter.next().toString(), equalTo(expectedSort1)); + assertThat(sortIter.next().toString(), equalTo(expectedSort2)); + String expectedSort = expectedSort1 + "," + expectedSort2; + String expected = "{\"size\":5,\"query\":{\"match\":{\"gender\":\"F\"}},\"sort\":[" + expectedSort + "]}"; + assertThat(node.toString(), equalTo(expected)); + // System.out.println(node.toString()); + // {"size":5,"query":{"match":{"gender":"F"}},"sort":[{"balance":"asc"},{"_uid":"desc"}]} + + ArrayNode token = JsonNodeFactory.instance.arrayNode(); + token.add(49223); + token.add(123); + expected = "{\"size\":6,\"query\":{\"match\":{\"gender\":\"F\"}},\"search_after\":[49223,123]," + + "\"sort\":[" + expectedSort + "]}"; + node = eSearch.elaborateQuery(query, token.toString(), 6); + + sizeNode = node.findValue("size"); + assertThat(sizeNode.asInt(), equalTo(6)); + ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); + assertThat(searchAfterNode.toString(), equalTo("[49223,123]")); + sortNode = node.findValue("sort"); + sortIter = sortNode.elements(); + assertThat(sortIter.next().toString(), equalTo(expectedSort1)); + assertThat(sortIter.next().toString(), equalTo(expectedSort2)); + assertThat(node.toString(), equalTo(expected)); + // System.out.println(node.toString()); + // {"size":6,"query":{"match":{"gender":"F"}},"search_after":[49223,123],"sort":[{"balance":"asc"},{"_uid":"desc"}]} } } diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java new file mode 100644 index 0000000000..f2c0038b0e --- /dev/null +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java @@ -0,0 +1,177 @@ +package nl.knaw.huygens.timbuctoo.search.elastic; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.core.IsEqual.equalTo; + +/** + * Created on 2017-10-04 11:13. + */ +public class PageableResultTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + public void readResult() throws Exception { + PageableResult pageableResult = new PageableResult(createQuery(), createResult()); + + String expectedQuery = "{\"size\":3,\"query\":{\"match\":{\"gender\":\"F\"}},\"search_after\":[1464," + + "\"account#174\"],\"sort\":[{\"balance\":\"asc\"},{\"_uid\":\"desc\"}]}"; + assertThat(pageableResult.getQuery(), equalTo(expectedQuery)); + + String expectedResultStart = "{\"took\":25,\"timed_out\":false"; + assertThat(pageableResult.getResult().startsWith(expectedResultStart), equalTo(true)); + + assertThat(pageableResult.getIdList().size(), equalTo(3)); + // System.out.println(pageableResult.getIdList()); + assertThat(pageableResult.getIdList(), contains("348", "490", "427")); + + String expectedToken = "[1463,\"account#427\"]"; + assertThat(pageableResult.getToken(), equalTo(expectedToken)); + + assertThat(pageableResult.getTotalHits(), equalTo(1000)); + assertThat(pageableResult.getSearchTime(), equalTo(25)); + } + + @Test + public void readEmptyResult() throws Exception { + PageableResult pageableResult = new PageableResult(null, createEmptyResult()); + + assertThat(pageableResult.getIdList().size(), equalTo(0)); + assertThat(pageableResult.getToken(), equalTo(null)); + assertThat(pageableResult.getTotalHits(), equalTo(0)); + assertThat(pageableResult.getSearchTime(), equalTo(5)); + } + + private JsonNode createQuery() throws IOException { + String queryString = "{\n" + + " \"size\": 3,\n" + + " \"query\": {\n" + + " \"match\" : {\n" + + " \"gender\" : \"F\"\n" + + " }\n" + + " },\n" + + " \"search_after\": [1464, \"account#174\"],\n" + + " \"sort\": [\n" + + " {\"balance\": \"asc\"},\n" + + " {\"_uid\": \"desc\"}\n" + + " ]\n" + + "}"; + return mapper.readTree(queryString); + } + + private JsonNode createResult() throws IOException { + String resultString = "{\n" + + " \"took\" : 25,\n" + + " \"timed_out\" : false,\n" + + " \"_shards\" : {\n" + + " \"total\" : 5,\n" + + " \"successful\" : 5,\n" + + " \"skipped\" : 0,\n" + + " \"failed\" : 0\n" + + " },\n" + + " \"hits\" : {\n" + + " \"total\" : 1000,\n" + + " \"max_score\" : null,\n" + + " \"hits\" : [\n" + + " {\n" + + " \"_index\" : \"bank\",\n" + + " \"_type\" : \"account\",\n" + + " \"_id\" : \"348\",\n" + + " \"_score\" : null,\n" + + " \"_source\" : {\n" + + " \"account_number\" : 348,\n" + + " \"balance\" : 1360,\n" + + " \"firstname\" : \"Karina\",\n" + + " \"lastname\" : \"Russell\",\n" + + " \"age\" : 37,\n" + + " \"gender\" : \"M\",\n" + + " \"address\" : \"797 Moffat Street\",\n" + + " \"employer\" : \"Limozen\",\n" + + " \"email\" : \"karinarussell@limozen.com\",\n" + + " \"city\" : \"Riegelwood\",\n" + + " \"state\" : \"RI\"\n" + + " },\n" + + " \"sort\" : [\n" + + " 1360,\n" + + " \"account#348\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"_index\" : \"bank\",\n" + + " \"_type\" : \"account\",\n" + + " \"_id\" : \"490\",\n" + + " \"_score\" : null,\n" + + " \"_source\" : {\n" + + " \"account_number\" : 490,\n" + + " \"balance\" : 1447,\n" + + " \"firstname\" : \"Strong\",\n" + + " \"lastname\" : \"Hendrix\",\n" + + " \"age\" : 26,\n" + + " \"gender\" : \"F\",\n" + + " \"address\" : \"134 Beach Place\",\n" + + " \"employer\" : \"Duoflex\",\n" + + " \"email\" : \"stronghendrix@duoflex.com\",\n" + + " \"city\" : \"Allentown\",\n" + + " \"state\" : \"ND\"\n" + + " },\n" + + " \"sort\" : [\n" + + " 1447,\n" + + " \"account#490\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"_index\" : \"bank\",\n" + + " \"_type\" : \"account\",\n" + + " \"_id\" : \"427\",\n" + + " \"_score\" : null,\n" + + " \"_source\" : {\n" + + " \"account_number\" : 427,\n" + + " \"balance\" : 1463,\n" + + " \"firstname\" : \"Rebekah\",\n" + + " \"lastname\" : \"Garrison\",\n" + + " \"age\" : 36,\n" + + " \"gender\" : \"F\",\n" + + " \"address\" : \"837 Hampton Avenue\",\n" + + " \"employer\" : \"Niquent\",\n" + + " \"email\" : \"rebekahgarrison@niquent.com\",\n" + + " \"city\" : \"Zarephath\",\n" + + " \"state\" : \"NY\"\n" + + " },\n" + + " \"sort\" : [\n" + + " 1463,\n" + + " \"account#427\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + return mapper.readTree(resultString); + } + + private JsonNode createEmptyResult() throws IOException { + String resultString = "{\n" + + " \"took\" : 5,\n" + + " \"timed_out\" : false,\n" + + " \"_shards\" : {\n" + + " \"total\" : 5,\n" + + " \"successful\" : 5,\n" + + " \"skipped\" : 0,\n" + + " \"failed\" : 0\n" + + " },\n" + + " \"hits\" : {\n" + + " \"total\" : 0,\n" + + " \"max_score\" : null,\n" + + " \"hits\" : [ ]\n" + + " }\n" + + "}"; + return mapper.readTree(resultString); + } + +} From 35fd2bdc86cae11ab0232119a7ce5f02a93c693b Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Wed, 4 Oct 2017 20:12:52 +0200 Subject: [PATCH 10/16] move to v5 --- .../{search/elastic => v5/elasticsearch}/ElasticSearch.java | 2 +- .../{search/elastic => v5/elasticsearch}/PageableResult.java | 2 +- .../elastic => v5/elasticsearch}/ElasticSearchOnLineTest.java | 2 +- .../{search/elastic => v5/elasticsearch}/ElasticSearchTest.java | 2 +- .../elastic => v5/elasticsearch}/PageableResultTest.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/{search/elastic => v5/elasticsearch}/ElasticSearch.java (98%) rename timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/{search/elastic => v5/elasticsearch}/PageableResult.java (95%) rename timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/{search/elastic => v5/elasticsearch}/ElasticSearchOnLineTest.java (98%) rename timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/{search/elastic => v5/elasticsearch}/ElasticSearchTest.java (98%) rename timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/{search/elastic => v5/elasticsearch}/PageableResultTest.java (99%) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java similarity index 98% rename from timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java rename to timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java index f87a4b312d..84fae76a9c 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java @@ -1,4 +1,4 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; +package nl.knaw.huygens.timbuctoo.v5.elasticsearch; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/PageableResult.java similarity index 95% rename from timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java rename to timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/PageableResult.java index c31c02e601..e1b3d7c787 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResult.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/PageableResult.java @@ -1,4 +1,4 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; +package nl.knaw.huygens.timbuctoo.v5.elasticsearch; import com.fasterxml.jackson.databind.JsonNode; diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearchOnLineTest.java similarity index 98% rename from timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java rename to timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearchOnLineTest.java index 7b8a4de8cc..4ed43fa27f 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchOnLineTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearchOnLineTest.java @@ -1,4 +1,4 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; +package nl.knaw.huygens.timbuctoo.v5.elasticsearch; import org.junit.BeforeClass; import org.junit.Test; diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearchTest.java similarity index 98% rename from timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java rename to timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearchTest.java index bd65b2ebef..08c71a9c6c 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/ElasticSearchTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearchTest.java @@ -1,4 +1,4 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; +package nl.knaw.huygens.timbuctoo.v5.elasticsearch; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/PageableResultTest.java similarity index 99% rename from timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java rename to timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/PageableResultTest.java index f2c0038b0e..57fc9eb582 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/search/elastic/PageableResultTest.java +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/PageableResultTest.java @@ -1,4 +1,4 @@ -package nl.knaw.huygens.timbuctoo.search.elastic; +package nl.knaw.huygens.timbuctoo.v5.elasticsearch; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; From 4bb418919d3db3e5069605d11967f3110534a63e Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Wed, 4 Oct 2017 20:50:57 +0200 Subject: [PATCH 11/16] integrate with timbuctoo --- Dockerfile | 5 ++ third_party/dependenciesGeneratedFor | 2 +- third_party/generate_workspace.bzl | 30 +++++++++++- timbuctoo-instancev4/docker_config.yaml | 7 +++ timbuctoo-instancev4/example_config.yaml | 6 +++ .../server/TimbuctooConfiguration.java | 3 ++ .../huygens/timbuctoo/server/TimbuctooV4.java | 4 +- .../java/nl/knaw/huygens/timbuctoo/v5/BUILD | 4 ++ .../v5/elasticsearch/ElasticSearch.java | 11 +++-- .../PaginationArgumentsHelper.java | 10 +++- .../datafetchers/RdfWiringFactory.java | 7 ++- .../datafetchers/CollectionDataFetcher.java | 48 +++++++++++++++---- .../datafetchers/dto/PaginationArguments.java | 10 +++- 13 files changed, 124 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2b099efe2..c86afde0f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,3 +19,8 @@ COPY ./timbuctoo-instancev4/docker_config.yaml ./timbuctoo-instancev4/docker_con CMD ["./timbuctoo-instancev4/target/appassembler/bin/timbuctoo", "server", "./timbuctoo-instancev4/docker_config.yaml"] RUN mkdir -p /root/data/dataSets + +ENV TIMBUCTOO_ELASTICSEARCH_HOST=http://example.com/elasticsearchhost +ENV TIMBUCTOO_ELASTICSEARCH_PORT=80 +ENV TIMBUCTOO_ELASTICSEARCH_USER=user +ENV TIMBUCTOO_ELASTICSEARCH_PASSWORD=password diff --git a/third_party/dependenciesGeneratedFor b/third_party/dependenciesGeneratedFor index 341c7241ed..ccacb72872 100644 --- a/third_party/dependenciesGeneratedFor +++ b/third_party/dependenciesGeneratedFor @@ -2,6 +2,6 @@ d34d24c8013df6412c755088853f531b1ffb5f20 ./HttpCommand/pom.xml 8180c8a378dbbbecf09271a55bdf72fbe63c55c2 ./pom.xml 6b1c1bbf94ffbf4124a58dede18d3916bad4b9ad ./security-client-agnostic/pom.xml -ec6efd34af68e1910237e5f9b2b67bd20aabcea1 ./timbuctoo-instancev4/pom.xml +ce0256285638e52833811467ee366f7e6eb21363 ./timbuctoo-instancev4/pom.xml d9c324fb24b3c7a11fc57cc2bb324fb4449a1e7f ./timbuctoo-test-services/pom.xml 021b25dc5b52fe7cd04089e7cde13d950c188add ./devtools/update_dependencies/dependencyparser.js diff --git a/third_party/generate_workspace.bzl b/third_party/generate_workspace.bzl index a1af6beb91..6bb2888207 100644 --- a/third_party/generate_workspace.bzl +++ b/third_party/generate_workspace.bzl @@ -390,7 +390,7 @@ def generated_maven_jars(): ) native.maven_jar( name = "commons_codec_commons_codec", - artifact = "commons-codec:commons-codec:jar:1.9", + artifact = "commons-codec:commons-codec:jar:1.10", ) native.maven_jar( name = "io_dropwizard_metrics_metrics_httpclient", @@ -649,6 +649,14 @@ def generated_maven_jars(): name = "com_healthmarketscience_jackcess_jackcess", artifact = "com.healthmarketscience.jackcess:jackcess:jar:2.1.6", ) + native.maven_jar( + name = "org_elasticsearch_client_rest", + artifact = "org.elasticsearch.client:rest:jar:5.5.3", + ) + native.maven_jar( + name = "org_apache_httpcomponents_httpasyncclient", + artifact = "org.apache.httpcomponents:httpasyncclient:jar:4.1.2", + ) native.maven_jar( name = "org_jsoup_jsoup", artifact = "org.jsoup:jsoup:jar:1.9.2", @@ -2617,6 +2625,26 @@ def generated_java_libraries(): ":commons_logging_commons_logging", ], ) + native.java_library( + name = "org_elasticsearch_client_rest", + visibility = ["//visibility:public"], + exports = ["@org_elasticsearch_client_rest//jar"], + runtime_deps = [ + ":org_apache_httpcomponents_httpclient", + ":org_apache_httpcomponents_httpcore", + ":org_apache_httpcomponents_httpasyncclient", + ":org_apache_httpcomponents_httpcore_nio", + ":commons_codec_commons_codec", + ":commons_logging_commons_logging", + ], + ) + native.java_library( + name = "org_apache_httpcomponents_httpasyncclient", + visibility = ["//visibility:public"], + exports = ["@org_apache_httpcomponents_httpasyncclient//jar"], + runtime_deps = [ + ], + ) native.java_library( name = "org_jsoup_jsoup", visibility = ["//visibility:public"], diff --git a/timbuctoo-instancev4/docker_config.yaml b/timbuctoo-instancev4/docker_config.yaml index 2bd904557f..d748fe25df 100644 --- a/timbuctoo-instancev4/docker_config.yaml +++ b/timbuctoo-instancev4/docker_config.yaml @@ -20,6 +20,13 @@ securityConfiguration: baseUri: ${BASE_URI} # used to generate next and previous links of a search result userRedirectUrl: ${TIMBUCTOO_GUI_PUBLIC_URL} +elasticSearch: + hostname: ${TIMBUCTOO_ELASTICSEARCH_HOST} + port: ${TIMBUCTOO_ELASTICSEARCH_PORT} + username: ${TIMBUCTOO_ELASTICSEARCH_USER} + password: ${TIMBUCTOO_ELASTICSEARCH_PASSWORD} + + archetypesSchema: | type Archetypes { persons: PersonArchetypeList @fromCollection(uri: "http://timbuctoo.huygens.knaw.nl/datasets/clusius/Persons", listAll: true) diff --git a/timbuctoo-instancev4/example_config.yaml b/timbuctoo-instancev4/example_config.yaml index 75cd3efac7..07633f8a0c 100644 --- a/timbuctoo-instancev4/example_config.yaml +++ b/timbuctoo-instancev4/example_config.yaml @@ -18,6 +18,12 @@ securityConfiguration: baseUri: http://127.0.0.1:0 #port will be replaced with the port that timbuctoo ends up listening on userRedirectUrl: http://example.com +elasticSearch: + hostname: http://example.com/elasticsearchinstance + port: 80 + username: foo + password: bar + archetypesSchema: | type Archetypes { persons: PersonArchetypeList @fromCollection(uri: "http://timbuctoo.huygens.knaw.nl/datasets/clusius/Persons", listAll: true) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooConfiguration.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooConfiguration.java index dbeb16b1c3..9d2899574b 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooConfiguration.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooConfiguration.java @@ -26,6 +26,7 @@ import nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.DataStoreCreationException; import nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbDataStoreFactory; import nl.knaw.huygens.timbuctoo.v5.datastores.resourcesync.ResourceSync; +import nl.knaw.huygens.timbuctoo.v5.elasticsearch.ElasticSearch; import nl.knaw.huygens.timbuctoo.v5.util.TimbuctooRdfIdHelper; import org.immutables.value.Value; @@ -109,6 +110,8 @@ public HttpClientConfiguration getHttpClientConfiguration() { @Valid public abstract DataSetConfiguration getDataSetConfiguration(); + public abstract ElasticSearch getElasticSearch(); + @JsonIgnore public DataSetRepository getDataSet() throws DataStoreCreationException { try { diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java index 1fd0cb6f4b..8643b24e9f 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java @@ -90,11 +90,11 @@ import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.CreateDataSet; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.DataSet; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.ErrorResponseHelper; +import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.GraphQl; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.JsonLdEditEndpoint; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.RdfUpload; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.ResourceSyncEndpoint; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.Rml; -import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.GraphQl; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.TabularUpload; import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.WellKnown; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.PaginationArgumentsHelper; @@ -279,7 +279,7 @@ public void run(TimbuctooConfiguration configuration, Environment environment) t dataSetRepository, serializerWriterRegistry, configuration.getArchetypesSchema(), - new RdfWiringFactory(dataSetRepository), + new RdfWiringFactory(dataSetRepository, configuration.getElasticSearch()), new DerivedSchemaTypeGenerator(new PaginationArgumentsHelper()) ), serializerWriterRegistry, diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/BUILD b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/BUILD index 34163719f6..ab788edafe 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/BUILD +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/BUILD @@ -27,6 +27,10 @@ java_library( "//timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/security", "//third_party:com_github_jsonld_java_jsonld_java", "//third_party:org_apache_httpcomponents_httpclient", + "//third_party:org_apache_httpcomponents_httpcore", + "//third_party:org_apache_httpcomponents_httpcore_nio", + "//third_party:org_apache_httpcomponents_httpasyncclient", + "//third_party:org_elasticsearch_client_rest", "//third_party:org_apache_jena_jena_core", "//third_party:com_google_code_findbugs_jsr305", "//third_party:org_apache_commons_commons_csv", diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java index 84fae76a9c..987438038b 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/elasticsearch/ElasticSearch.java @@ -1,5 +1,7 @@ package nl.knaw.huygens.timbuctoo.v5.elasticsearch; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -23,9 +25,6 @@ import java.util.Collections; import java.util.Map; -/** - * Created on 2017-09-25 15:28. - */ public class ElasticSearch { public static final String UNIQUE_FIELD_NAME = "_uid"; @@ -35,7 +34,9 @@ public class ElasticSearch { private final RestClient restClient; private final ObjectMapper mapper; - public ElasticSearch(String hostname, int port, String username, String password) { + @JsonCreator + public ElasticSearch(@JsonProperty("hostname") String hostname, @JsonProperty("port") int port, + @JsonProperty("username") String username, @JsonProperty("password") String password) { Header[] headers = { new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), new BasicHeader("Role", "Read")}; @@ -74,7 +75,7 @@ protected ObjectNode elaborateQuery(String elasticSearchQuery, String token, int node.put("size", preferredPageSize); // search_after - if (token != null) { + if (token != null && !token.isEmpty()) { ArrayNode searchAfterNode = (ArrayNode) node.findValue("search_after"); if (searchAfterNode == null) { searchAfterNode = node.putArray("search_after"); diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/PaginationArgumentsHelper.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/PaginationArgumentsHelper.java index 9963fc66d5..8f235a0573 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/PaginationArgumentsHelper.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/PaginationArgumentsHelper.java @@ -8,6 +8,7 @@ public class PaginationArgumentsHelper { static PaginationArguments getPaginationArguments(DataFetchingEnvironment environment) { String cursor = ""; + String searchQuery = null; int count = DEFAULT_COUNT; if (environment.containsArgument("cursor")) { cursor = environment.getArgument("cursor"); @@ -17,7 +18,12 @@ static PaginationArguments getPaginationArguments(DataFetchingEnvironment enviro count = environment.getArgument("count"); } - return PaginationArguments.create(count, cursor); + if (environment.containsArgument("elasticsearch")) { + searchQuery = environment.getArgument("elasticsearch"); + } + + + return PaginationArguments.create(count, cursor, searchQuery); } public String makeListName(String outputTypeName) { @@ -33,7 +39,7 @@ public String makeListField(String fieldName, String outputTypeName) { // .name("count") // .description("The amount of items to request. You might get less items then requested.") // ); - return fieldName + "(cursor: ID, count: Int): " + makeListName(outputTypeName); + return fieldName + "(cursor: ID, count: Int, elasticsearch: String): " + makeListName(outputTypeName); } public String makePaginatedListDefinition(String outputType) { diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/RdfWiringFactory.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/RdfWiringFactory.java index ed3887342f..43497064ec 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/RdfWiringFactory.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/RdfWiringFactory.java @@ -19,6 +19,7 @@ import nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet; import nl.knaw.huygens.timbuctoo.v5.datastores.prefixstore.TypeNameStore; import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction; +import nl.knaw.huygens.timbuctoo.v5.elasticsearch.ElasticSearch; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.datafetchers.CollectionDataFetcher; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.datafetchers.QuadStoreLookUpSubjectByUriFetcher; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.datafetchers.RelationDataFetcher; @@ -41,9 +42,11 @@ public class RdfWiringFactory implements WiringFactory { private final LookUpSubjectByUriFetcherWrapper lookupFetcher; private final ObjectTypeResolver objectTypeResolver; private final DataSetRepository dataSetRepository; + private final ElasticSearch elasticSearch; - public RdfWiringFactory(DataSetRepository dataSetRepository) { + public RdfWiringFactory(DataSetRepository dataSetRepository, ElasticSearch elasticSearch) { this.dataSetRepository = dataSetRepository; + this.elasticSearch = elasticSearch; objectTypeResolver = new ObjectTypeResolver(); uriFetcher = new UriFetcher(); lookupFetcher = new LookUpSubjectByUriFetcherWrapper("uri", new QuadStoreLookUpSubjectByUriFetcher()); @@ -98,7 +101,7 @@ public DataFetcher getDataFetcher(FieldWiringEnvironment environment) { String uri = ((StringValue) directive.getArgument("uri").getValue()).getValue(); boolean listAll = ((BooleanValue) directive.getArgument("listAll").getValue()).isValue(); if (listAll) { - return new CollectionFetcherWrapper(new CollectionDataFetcher(uri)); + return new CollectionFetcherWrapper(new CollectionDataFetcher(uri, elasticSearch)); } else { return lookupFetcher; } diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/berkeleydb/datafetchers/CollectionDataFetcher.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/berkeleydb/datafetchers/CollectionDataFetcher.java index 062ede2983..62d7313694 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/berkeleydb/datafetchers/CollectionDataFetcher.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/berkeleydb/datafetchers/CollectionDataFetcher.java @@ -1,34 +1,64 @@ package nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.datafetchers; +import graphql.GraphQLError; import nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet; import nl.knaw.huygens.timbuctoo.v5.datastores.collectionindex.CursorSubject; +import nl.knaw.huygens.timbuctoo.v5.elasticsearch.ElasticSearch; +import nl.knaw.huygens.timbuctoo.v5.elasticsearch.PageableResult; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.CollectionFetcher; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.dto.LazyTypeSubjectReference; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.PaginatedList; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.PaginationArguments; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.SubjectReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.stream.Collectors; import java.util.stream.Stream; -import static nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.datafetchers.PaginationHelper.getPaginatedList; +import static nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.datafetchers.PaginationHelper + .getPaginatedList; public class CollectionDataFetcher implements CollectionFetcher { + private static final Logger LOG = LoggerFactory.getLogger(CollectionDataFetcher.class); + private final String collectionName; + private final ElasticSearch elasticSearch; - public CollectionDataFetcher(String collectionName) { + public CollectionDataFetcher(String collectionName, ElasticSearch elasticSearch) { this.collectionName = collectionName; + this.elasticSearch = elasticSearch; } @Override public PaginatedList getList(PaginationArguments arguments, DataSet dataSet) { String cursor = arguments.getCursor(); - try (Stream subjectStream = dataSet.getCollectionIndex().getSubjects(collectionName, cursor)) { - return getPaginatedList( - subjectStream, - cursorSubject -> new LazyTypeSubjectReference(cursorSubject.getSubjectUri(), dataSet), - arguments.getCount(), - !cursor.isEmpty() - ); + if (arguments.getSearchQuery().isPresent()) { + try { + final PageableResult result = elasticSearch.query( + collectionName, + arguments.getSearchQuery().get(), + cursor, + arguments.getCount() + ); + return PaginatedList.create( + null, + result.getToken(), + result.getIdList().stream().map(x -> new LazyTypeSubjectReference(x, dataSet)).collect(Collectors.toList()) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + try (Stream subjectStream = dataSet.getCollectionIndex().getSubjects(collectionName, cursor)) { + return getPaginatedList( + subjectStream, + cursorSubject -> new LazyTypeSubjectReference(cursorSubject.getSubjectUri(), dataSet), + arguments.getCount(), + !cursor.isEmpty() + ); + } } } diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/PaginationArguments.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/PaginationArguments.java index b523d62f2e..97fa097796 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/PaginationArguments.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/PaginationArguments.java @@ -4,6 +4,7 @@ import org.immutables.value.Value; import java.util.Base64; +import java.util.Optional; @Value.Immutable public interface PaginationArguments { @@ -19,15 +20,22 @@ public interface PaginationArguments { */ String getCursor(); + + /** + * SearchQuery is a String-serialized elasticsearch query + */ + Optional getSearchQuery(); + /** * Count is either a positive number which provides a suggestion on how many items to return (might be overridden by * the data provider) or a negative number (usually -1) that indicates that there is no preference */ int getCount(); - static PaginationArguments create(int count, String cursor) { + static PaginationArguments create(int count, String cursor, String searchQuery) { return ImmutablePaginationArguments.builder() .count(count) + .searchQuery(Optional.ofNullable(searchQuery)) .cursor(new String(DECODER.decode(cursor), Charsets.UTF_8)) .build(); } From a7309246ea9bb4c1aa0130baa59a2ace3d6a57b1 Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Thu, 5 Oct 2017 08:10:16 +0200 Subject: [PATCH 12/16] clean up and add IndexConfig --- .../huygens/timbuctoo/server/TimbuctooV4.java | 3 +- .../v5/dataset/DataSetRepository.java | 32 ++-- .../v5/dataset/dto/PromotedDataSet.java | 6 + .../v5/dropwizard/endpoints/GraphQl.java | 4 +- .../datafetchers/dto/DataSetWithDatabase.java | 44 ++++++ .../v5/graphql/datafetchers/dto/RootData.java | 17 +++ .../v5/graphql/rootquery/RootQuery.java | 138 +++++++----------- .../timbuctoo/v5/util/RdfConstants.java | 2 +- .../v5/graphql/rootquery/schema.graphql | 30 +++- 9 files changed, 164 insertions(+), 112 deletions(-) create mode 100644 timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/DataSetWithDatabase.java create mode 100644 timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/RootData.java diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java index 8643b24e9f..ce5559ded4 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java @@ -280,7 +280,8 @@ public void run(TimbuctooConfiguration configuration, Environment environment) t serializerWriterRegistry, configuration.getArchetypesSchema(), new RdfWiringFactory(dataSetRepository, configuration.getElasticSearch()), - new DerivedSchemaTypeGenerator(new PaginationArgumentsHelper()) + new DerivedSchemaTypeGenerator(new PaginationArgumentsHelper()), + environment.getObjectMapper() ), serializerWriterRegistry, securityConfig.getLoggedInUsers() diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/DataSetRepository.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/DataSetRepository.java index 8861d16822..d98b294f89 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/DataSetRepository.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/DataSetRepository.java @@ -111,6 +111,12 @@ public Optional getDataSet(String ownerId, String dataSetId) { } } + public Optional getDataSet(String combinedId) { + final Tuple splitId = PromotedDataSet.splitCombinedId(combinedId); + return Optional.ofNullable(dataSetMap.get(splitId.getLeft())) + .map(userDataSets -> userDataSets.get(splitId.getRight())); + } + public DataSet createDataSet(String ownerId, String dataSetId) throws DataStoreCreationException { final PromotedDataSet dataSet = promotedDataSet(ownerId, dataSetId, rdfIdHelper.dataSet(ownerId, dataSetId), false); synchronized (dataSetMap) { @@ -141,26 +147,18 @@ public boolean dataSetExists(String ownerId, String dataSet) { return getDataSet(ownerId, dataSet).isPresent(); } - public Map> getDataSets() { - return storedDataSets.getData(); + public Collection getDataSets() { + return dataSetMap.values().stream().flatMap(x -> x.values().stream()).collect(Collectors.toList()); } - public Map> getPromotedDataSets() { - Map> dataSets = storedDataSets.getData(); - Map> promotedDataSets = new HashMap<>(); - - for (Map.Entry> userDataSets : dataSets.entrySet()) { - Set mappedUserSets = userDataSets.getValue() - .stream() - .filter(dataSet -> dataSet.isPromoted()) - .collect(Collectors.toSet()); - promotedDataSets.put(userDataSets.getKey(), mappedUserSets); - } - return promotedDataSets; + public Collection getPromotedDataSets() { + return dataSetMap.values().stream().flatMap(x -> x.values().stream()) + .filter(x -> x.getMetadata().isPromoted()) + .collect(Collectors.toList()); } - public Collection getDataSetsWithWriteAccess(String userId) { - List dataSetsWithWriteAccess = new ArrayList<>(); + public Collection getDataSetsWithWriteAccess(String userId) { + List dataSetsWithWriteAccess = new ArrayList<>(); for (Map userDataSets : dataSetMap.values()) { for (DataSet dataSet : userDataSets.values()) { @@ -170,7 +168,7 @@ public Collection getDataSetsWithWriteAccess(String userId) { .map(VreAuthorization::isAllowedToWrite) .orElse(false); if (isAllowedToWrite) { - dataSetsWithWriteAccess.add(dataSet.getMetadata()); + dataSetsWithWriteAccess.add(dataSet); } } catch (AuthorizationUnavailableException e) { LOG.error("Could not fetch authorization", e); diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/dto/PromotedDataSet.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/dto/PromotedDataSet.java index 1463906ee6..259670edab 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/dto/PromotedDataSet.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/dto/PromotedDataSet.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import nl.knaw.huygens.timbuctoo.util.Tuple; import org.immutables.value.Value; import java.util.Optional; @@ -26,6 +27,11 @@ public interface PromotedDataSet { Optional role = Optional.empty(); + static Tuple splitCombinedId(String combinedId) { + String[] parts = combinedId.split("__", 2); + return Tuple.tuple(parts[0], parts[1]); + } + static PromotedDataSet promotedDataSet(String ownerId, String dataSetId, String baseUri, boolean promoted) { if (!ownerId.matches(VALID_ID) || !dataSetId.matches(VALID_ID)) { throw new IllegalArgumentException("Owner id and dataSet id should match " + VALID_ID); diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/GraphQl.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/GraphQl.java index 13697d9286..c8b5b320b4 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/GraphQl.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/GraphQl.java @@ -11,7 +11,7 @@ import nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException; import nl.knaw.huygens.timbuctoo.v5.dropwizard.contenttypes.SerializerWriter; import nl.knaw.huygens.timbuctoo.v5.dropwizard.contenttypes.SerializerWriterRegistry; -import nl.knaw.huygens.timbuctoo.v5.graphql.rootquery.RootQuery; +import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.RootData; import nl.knaw.huygens.timbuctoo.v5.graphql.serializable.SerializerExecutionStrategy; import nl.knaw.huygens.timbuctoo.v5.serializable.SerializableResult; @@ -147,7 +147,7 @@ public Response executeGraphql(String query, String acceptHeader, String acceptP } final ExecutionResult result = graphQl .execute(newExecutionInput() - .root(new RootQuery.RootData(loggedInUsers.userFor(authHeader))) + .root(new RootData(loggedInUsers.userFor(authHeader))) .query(queryFromBody) .operationName(operationName) .variables(variables == null ? Collections.emptyMap() : variables) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/DataSetWithDatabase.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/DataSetWithDatabase.java new file mode 100644 index 0000000000..866188f075 --- /dev/null +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/DataSetWithDatabase.java @@ -0,0 +1,44 @@ +package nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto; + +import nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet; +import nl.knaw.huygens.timbuctoo.v5.dataset.dto.PromotedDataSet; +import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.dto.LazyTypeSubjectReference; + +public class DataSetWithDatabase extends LazyTypeSubjectReference implements PromotedDataSet { + private final PromotedDataSet promotedDataSet; + + @Override + public String getDataSetId() { + return promotedDataSet.getDataSetId(); + } + + @Override + public String getOwnerId() { + return promotedDataSet.getOwnerId(); + } + + @Override + public String getBaseUri() { + return promotedDataSet.getBaseUri(); + } + + @Override + public String getCombinedId() { + return promotedDataSet.getCombinedId(); + } + + public boolean isPromoted() { + return promotedDataSet.isPromoted(); + } + + public DataSetWithDatabase(DataSet dataSet) { + super(dataSet.getMetadata().getBaseUri(), dataSet); + this.promotedDataSet = dataSet.getMetadata(); + } + + @Override + public String getSubjectUri() { + return promotedDataSet.getBaseUri(); + } + +} diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/RootData.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/RootData.java new file mode 100644 index 0000000000..dd797390ef --- /dev/null +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/datafetchers/dto/RootData.java @@ -0,0 +1,17 @@ +package nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto; + +import nl.knaw.huygens.timbuctoo.security.dto.User; + +import java.util.Optional; + +public class RootData { + public Optional getCurrentUser() { + return currentUser; + } + + private final Optional currentUser; + + public RootData(Optional currentUser) { + this.currentUser = currentUser; + } +} diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/RootQuery.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/RootQuery.java index 4fe8ca273d..2ec1ed4ae4 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/RootQuery.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/RootQuery.java @@ -1,5 +1,6 @@ package nl.knaw.huygens.timbuctoo.v5.graphql.rootquery; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.io.Resources; import graphql.schema.GraphQLSchema; @@ -12,10 +13,15 @@ import nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet; import nl.knaw.huygens.timbuctoo.v5.dataset.dto.PromotedDataSet; import nl.knaw.huygens.timbuctoo.v5.datastores.prefixstore.TypeNameStore; +import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.QuadStore; +import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.CursorQuad; +import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction; import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Type; import nl.knaw.huygens.timbuctoo.v5.dropwizard.SupportedExportFormats; import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.RdfWiringFactory; -import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.berkeleydb.dto.LazyTypeSubjectReference; +import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.DataSetWithDatabase; +import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.RootData; +import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.SubjectReference; import nl.knaw.huygens.timbuctoo.v5.graphql.derivedschema.DerivedSchemaTypeGenerator; import nl.knaw.huygens.timbuctoo.v5.graphql.rootquery.dataproviders.CollectionMetadata; import nl.knaw.huygens.timbuctoo.v5.graphql.rootquery.dataproviders.CollectionMetadataList; @@ -27,35 +33,46 @@ import nl.knaw.huygens.timbuctoo.v5.graphql.rootquery.dataproviders.MimeTypeDescription; import nl.knaw.huygens.timbuctoo.v5.graphql.rootquery.dataproviders.Property; import nl.knaw.huygens.timbuctoo.v5.util.RdfConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Collection; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.google.common.io.Resources.getResource; +import static nl.knaw.huygens.timbuctoo.v5.util.RdfConstants.TIM_HASINDEXERCONFIG; public class RootQuery implements Supplier { + private static final Logger LOG = LoggerFactory.getLogger(RootQuery.class); + + private final DataSetRepository dataSetRepository; private final SupportedExportFormats supportedFormats; private final String archetypes; private final RdfWiringFactory wiringFactory; private final DerivedSchemaTypeGenerator typeGenerator; + private final ObjectMapper objectMapper; private final SchemaParser schemaParser; private final String staticQuery; public RootQuery(DataSetRepository dataSetRepository, SupportedExportFormats supportedFormats, String archetypes, - RdfWiringFactory wiringFactory, DerivedSchemaTypeGenerator typeGenerator) throws IOException { + RdfWiringFactory wiringFactory, DerivedSchemaTypeGenerator typeGenerator, ObjectMapper objectMapper) + throws IOException { this.dataSetRepository = dataSetRepository; this.supportedFormats = supportedFormats; this.archetypes = archetypes; this.wiringFactory = wiringFactory; this.typeGenerator = typeGenerator; + this.objectMapper = objectMapper; staticQuery = Resources.toString(getResource(RootQuery.class, "schema.graphql"), Charsets.UTF_8); schemaParser = new SchemaParser(); } @@ -77,23 +94,16 @@ public synchronized GraphQLSchema rebuildSchema() { wiring.type("Query", builder -> builder .dataFetcher("promotedDataSets", env -> dataSetRepository.getPromotedDataSets() - .values().stream().flatMap(Collection::stream) - .map(this::makeDbResult) + .stream() + .map(DataSetWithDatabase::new) .collect(Collectors.toList())) .dataFetcher("allDataSets", env -> dataSetRepository.getDataSets() - .values().stream().flatMap(Collection::stream) - .map(this::makeDbResult) + .stream() + .map(DataSetWithDatabase::new) .collect(Collectors.toList())) .dataFetcher("dataSetMetadata", env -> { - String[] parsedId = ((String) env.getArgument("dataSetId")).split("__", 2); - return Optional.ofNullable(dataSetRepository.getDataSets().get(parsedId[0])).map(d -> { - for (PromotedDataSet promotedDataSet : d) { - if (promotedDataSet.getDataSetId().equals(parsedId[1])) { - return makeDbResult(promotedDataSet); - } - } - return null; - }); + final String dataSetId = env.getArgument("dataSetId"); + return dataSetRepository.getDataSet(dataSetId).map(DataSetWithDatabase::new); }) .dataFetcher("aboutMe", env -> ((RootData) env.getRoot()).getCurrentUser().orElse(null)) .dataFetcher("availableExportMimetypes", env -> supportedFormats.getSupportedMimeTypes().stream() @@ -121,11 +131,36 @@ public synchronized GraphQLSchema rebuildSchema() { }) .dataFetcher("dataSetId", env -> ((PromotedDataSet) env.getSource()).getCombinedId()) ); + wiring.type("CollectionMetadata", builder -> builder + .dataFetcher("indexConfig", env -> { + SubjectReference source = env.getSource(); + final QuadStore qs = source.getDataSet().getQuadStore(); + try (Stream quads = qs.getQuads(source.getSubjectUri(), TIM_HASINDEXERCONFIG, Direction.OUT, "")) { + final Map result = quads.findFirst() + .map(q -> { + try { + return objectMapper.readValue(q.getObject(), Map.class); + } catch (IOException e) { + LOG.error("Value not a Map", e); + return new HashMap<>(); + } + }) + .orElse(new HashMap()); + if (!result.containsKey("facet") || !(result.get("facet") instanceof List)) { + result.put("facet", new ArrayList<>()); + } + if (!result.containsKey("fullText") || !(result.get("fullText") instanceof List)) { + result.put("fullText", new ArrayList<>()); + } + return result; + } + }) + ); wiring.type("AboutMe", builder -> builder .dataFetcher("dataSets", env -> (Iterable) () -> dataSetRepository .getDataSetsWithWriteAccess(((User) env.getSource()).getPersistentId()) - .stream().map(this::makeDbResult).iterator() + .stream().map(DataSetWithDatabase::new).iterator() ) .dataFetcher("id", env -> ((User) env.getSource()).getPersistentId()) .dataFetcher("name", env -> ((User) env.getSource()).getDisplayName()) @@ -137,13 +172,10 @@ public synchronized GraphQLSchema rebuildSchema() { StringBuilder root = new StringBuilder("type DataSets {\n"); boolean[] dataSetAvailable = new boolean[] {false}; - dataSetRepository.getDataSets().values().stream().flatMap(Collection::stream).forEach(promotedDataSet -> { + dataSetRepository.getDataSets().forEach(dataSet -> { + final PromotedDataSet promotedDataSet = dataSet.getMetadata(); final String name = promotedDataSet.getCombinedId(); - final DataSet dataSet = dataSetRepository.getDataSet( - promotedDataSet.getOwnerId(), - promotedDataSet.getDataSetId() - ).get(); final Map types = dataSet.getSchemaStore().getTypes(); if (types != null) { dataSetAvailable[0] = true; @@ -158,7 +190,7 @@ public synchronized GraphQLSchema rebuildSchema() { .append("\")\n"); wiring.type(name, c -> c - .dataFetcher("metadata", env -> makeDbResult(promotedDataSet)) + .dataFetcher("metadata", env -> new DataSetWithDatabase(dataSet)) ); final String schema = typeGenerator.makeGraphQlTypes( @@ -179,16 +211,6 @@ public synchronized GraphQLSchema rebuildSchema() { return schemaGenerator.makeExecutableSchema(staticQuery, wiring.build()); } - public DataSetWithDatabase makeDbResult(PromotedDataSet promotedDataSet) { - return new DataSetWithDatabase( - dataSetRepository.getDataSet( - promotedDataSet.getOwnerId(), - promotedDataSet.getDataSetId() - ).get(), - promotedDataSet - ); - } - public CollectionMetadataList getCollections(PromotedDataSet input) { final DataSet dataSet = dataSetRepository.getDataSet(input.getOwnerId(), input.getDataSetId()).get(); @@ -258,54 +280,4 @@ public GraphQLSchema get() { return rebuildSchema(); } - public static class RootData { - public Optional getCurrentUser() { - return currentUser; - } - - private final Optional currentUser; - - public RootData(Optional currentUser) { - this.currentUser = currentUser; - } - } - - private static class DataSetWithDatabase extends LazyTypeSubjectReference implements PromotedDataSet { - private final PromotedDataSet promotedDataSet; - - @Override - public String getDataSetId() { - return promotedDataSet.getDataSetId(); - } - - @Override - public String getOwnerId() { - return promotedDataSet.getOwnerId(); - } - - @Override - public String getBaseUri() { - return promotedDataSet.getBaseUri(); - } - - @Override - public String getCombinedId() { - return promotedDataSet.getCombinedId(); - } - - public boolean isPromoted() { - return promotedDataSet.isPromoted(); - } - - public DataSetWithDatabase(DataSet dataSet, PromotedDataSet promotedDataSet) { - super(promotedDataSet.getBaseUri(), dataSet); - this.promotedDataSet = promotedDataSet; - } - - @Override - public String getSubjectUri() { - return promotedDataSet.getBaseUri(); - } - - } } diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/util/RdfConstants.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/util/RdfConstants.java index 10b6a86429..f7bc2d1a2b 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/util/RdfConstants.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/util/RdfConstants.java @@ -24,7 +24,7 @@ public class RdfConstants { public static final String TIM_MIMETYPE = TIM_VOCAB + "mimetype"; public static final String TIM_HASCOLOR = TIM_VOCAB + "hasColor"; public static final String TIM_EDITOR = TIM_VOCAB + "editor"; - public static final String TIM_HASINDEXERCONFIG = TIM_VOCAB + "hasIndexerConfig"; + public static final String TIM_HASINDEXERCONFIG = TIM_VOCAB + "hasIndexConfig"; public static final String TIM_HASFULLTEXTSEARCH = TIM_VOCAB + "hasFullTextSearch"; public static final String TIM_HASFACETPATH = TIM_VOCAB + "hasFacetPath"; public static final String TIM_NEXTFACET = TIM_VOCAB + "nextFacet"; diff --git a/timbuctoo-instancev4/src/main/resources/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/schema.graphql b/timbuctoo-instancev4/src/main/resources/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/schema.graphql index e1bd5560a8..c897e23999 100644 --- a/timbuctoo-instancev4/src/main/resources/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/schema.graphql +++ b/timbuctoo-instancev4/src/main/resources/nl/knaw/huygens/timbuctoo/v5/graphql/rootquery/schema.graphql @@ -1,5 +1,5 @@ schema { - query: Query + query: Query, } interface Value { @@ -85,7 +85,7 @@ type CollectionMetadata { title: Value @rdf(predicate: "http://www.w3.org/2000/01/rdf-schema#label", direction: "OUT", isList: false, isObject: false, isValue: true) archeType: Entity @rdf(predicate: "http://www.w3.org/2000/01/rdf-schema#subClassOf", direction: "OUT", isList: false, isObject: true, isValue: false) - indexerConfig: IndexerConfig @rdf(predicate: "http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#hasIndexerConfig", direction: "OUT", isList: false, isObject: true, isValue: false) + indexConfig: IndexConfig! summaryProperties: SummaryProperties! @passThrough } @@ -95,11 +95,25 @@ type SummaryProperties { image: Value @rdf(predicate: "http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#summaryImagePredicate", direction: "OUT", isList: false, isObject: false, isValue: true) } -type IndexerConfig { - path: Value @rdf(predicate: "http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#hasFacetPath", direction: "OUT", isList: false, isObject: false, isValue: true) - type: Value @rdf(predicate: "http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#hasFacetType", direction: "OUT", isList: false, isObject: false, isValue: true) - fullTextSearch: Value @rdf(predicate: "http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#hasFullTextSearch", direction: "OUT", isList: false, isObject: false, isValue: true) - next: IndexerConfig @rdf(predicate: "http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#nextFacet", direction: "OUT", isList: false, isObject: true, isValue: false) +type IndexConfig { + facet: [FacetConfig]! + fullText: [FullTextConfig]! +} + +type FacetConfig { + paths: [String] + type: String + caption: String +} + +type FullTextConfig { + caption: String + fields: [FullTextFieldConfig] +} + +type FullTextFieldConfig { + path: String + boost: Float } type PropertyList { @@ -134,4 +148,4 @@ type License { type ProvenanceInfo { title: Value @rdf(predicate: "http://purl.org/dc/terms/title", direction: "OUT", isList: false, isObject: false, isValue: true) body: Value @rdf(predicate: "http://purl.org/dc/terms/description", direction: "OUT", isList: false, isObject: false, isValue: true) -} \ No newline at end of file +} From ec8b9546e4eadbd1a2030551e35cf469d05069ab Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Thu, 5 Oct 2017 08:50:23 +0200 Subject: [PATCH 13/16] redirect to graphiql by default --- timbuctoo-instancev4/docker_config.yaml | 2 +- timbuctoo-instancev4/example_config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/timbuctoo-instancev4/docker_config.yaml b/timbuctoo-instancev4/docker_config.yaml index d748fe25df..434f195e35 100644 --- a/timbuctoo-instancev4/docker_config.yaml +++ b/timbuctoo-instancev4/docker_config.yaml @@ -18,7 +18,7 @@ securityConfiguration: authenticationCredentials: DUMMY baseUri: ${BASE_URI} # used to generate next and previous links of a search result -userRedirectUrl: ${TIMBUCTOO_GUI_PUBLIC_URL} +userRedirectUrl: ${BASE_URI}/static/graphiql elasticSearch: hostname: ${TIMBUCTOO_ELASTICSEARCH_HOST} diff --git a/timbuctoo-instancev4/example_config.yaml b/timbuctoo-instancev4/example_config.yaml index 07633f8a0c..5e52116e86 100644 --- a/timbuctoo-instancev4/example_config.yaml +++ b/timbuctoo-instancev4/example_config.yaml @@ -16,7 +16,7 @@ securityConfiguration: authenticationCredentials: DUMMY baseUri: http://127.0.0.1:0 #port will be replaced with the port that timbuctoo ends up listening on -userRedirectUrl: http://example.com +userRedirectUrl: ./static/graphiql elasticSearch: hostname: http://example.com/elasticsearchinstance From 46af047f8bf4f95819c25b0b1e4aefd380b87998 Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Thu, 5 Oct 2017 09:01:46 +0200 Subject: [PATCH 14/16] fix test dependencies for bazel --- .../src/test/java/nl/knaw/huygens/timbuctoo/v5/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/BUILD b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/BUILD index 52fdf053bf..8edf7dd4b2 100644 --- a/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/BUILD +++ b/timbuctoo-instancev4/src/test/java/nl/knaw/huygens/timbuctoo/v5/BUILD @@ -14,6 +14,7 @@ junit_suite_test( "//third_party:org_glassfish_jaxb_txw2", + "//third_party:org_elasticsearch_client_rest", "//timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/util", "//timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/rml", "//timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/rml/jena", From 20f2bd528113d6a453ae1c873cc98a5aca31d22b Mon Sep 17 00:00:00 2001 From: Jauco Noordzij Date: Thu, 5 Oct 2017 10:22:17 +0200 Subject: [PATCH 15/16] add default indexer config --- .../nl/knaw/huygens/timbuctoo/bia_clusius.ttl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/timbuctoo-instancev4/src/test/resources/nl/knaw/huygens/timbuctoo/bia_clusius.ttl b/timbuctoo-instancev4/src/test/resources/nl/knaw/huygens/timbuctoo/bia_clusius.ttl index 1836dcb18d..3a6a65871c 100644 --- a/timbuctoo-instancev4/src/test/resources/nl/knaw/huygens/timbuctoo/bia_clusius.ttl +++ b/timbuctoo-instancev4/src/test/resources/nl/knaw/huygens/timbuctoo/bia_clusius.ttl @@ -24,15 +24,21 @@ <./provenance> "Example Provenance Title" . <./provenance> "This is an example description for the example provenance." . - -clusius:Places +clusius:Places rdfs:subClassOf archetypes:location ; + "{\"facet\":[{\"paths\":[\"tim_name.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_country.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_latitude.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_longitude.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_remarks.value\"],\"type\":\"MultiSelect\"}]}" ; "tim_name" . clusius:Persons rdfs:subClassOf archetypes:person ; + "{\"facet\":[{\"paths\":[\"tim_names.items.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_gender.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_birthDate.value\"],\"type\":\"DateRange\"},{\"paths\":[\"tim_deathDate.value\"],\"type\":\"DateRange\"},{\"paths\":[\"tim_hasDeathPlace.tim_country.value\",\"tim_hasDeathPlace.tim_name.value\"],\"type\":\"Hierarchical\"},{\"paths\":[\"tim_hasBirthPlace.tim_country.value\",\"tim_hasBirthPlace.tim_name.value\"],\"type\":\"Hierarchical\"},{\"paths\":[\"_inverse_tim_isScientistBioOf.tim_biography.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"_inverse_tim_hasResident.items.tim_hasLocation.tim_country.value\",\"_inverse_tim_hasResident.items.tim_hasLocation.tim_name.value\"],\"type\":\"Hierarchical\"},{\"paths\":[\"_inverse_tim_isEducationOf.items.tim_description.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"_inverse_tim_isOccupationOf.items.tim_description.value\"],\"type\":\"MultiSelect\"}],\"fullText\":[{\"fields\":[{\"path\":\"tim_names.items.value\"}]}]}" ; "tim_names" . +clusius:Biography + rdfs:subClassOf archetypes:document ; + "{\"facet\":[{\"paths\":[\"tim_isScientistBioOf.tim_names.items.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_hasFieldOfInterest.items.tim_value.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_biography.value\"],\"type\":\"MultiSelect\"},{\"paths\":[\"tim_highest_degree.value\"],\"type\":\"MultiSelect\"}]}" ; + "tim_biography" . + clusius:Provenance_type rdfs:subClassOf archetypes:keyword . clusius:Fields_of_interest rdfs:subClassOf archetypes:keyword . clusius:Publications rdfs:subClassOf archetypes:document . From b821d98f4ed1fcc53861c539f28675451b320e3a Mon Sep 17 00:00:00 2001 From: Martijn Maas Date: Fri, 6 Oct 2017 14:38:45 +0200 Subject: [PATCH 16/16] Fix typo --- .../huygens/timbuctoo/v5/dropwizard/endpoints/WellKnown.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/WellKnown.java b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/WellKnown.java index 25fc3c9166..25f985aa80 100644 --- a/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/WellKnown.java +++ b/timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/WellKnown.java @@ -10,7 +10,7 @@ public class WellKnown { @Path("resourcesync") @GET - public Response resoureSync() { + public Response resourceSync() { // Permanent redirect return Response.seeOther(UriBuilder.fromResource(ResourceSyncEndpoint.class) .path(ResourceSyncEndpoint.SOURCE_DESCRIPTION_PATH)