Merge branch 'master' of https://github.com/HuygensING/timbuctoo into…

… sql-export

Dockerfile

@@ -19,3 +19,8 @@ COPY ./timbuctoo-instancev4/docker_config.yaml ./timbuctoo-instancev4/docker_con
 CMD ["./timbuctoo-instancev4/target/appassembler/bin/timbuctoo", "server", "./timbuctoo-instancev4/docker_config.yaml"]
 
 RUN mkdir -p /root/data/dataSets
+
+ENV TIMBUCTOO_ELASTICSEARCH_HOST=http://example.com/elasticsearchhost
+ENV TIMBUCTOO_ELASTICSEARCH_PORT=80
+ENV TIMBUCTOO_ELASTICSEARCH_USER=user
+ENV TIMBUCTOO_ELASTICSEARCH_PASSWORD=password

third_party/dependenciesGeneratedFor

@@ -2,6 +2,6 @@
 d34d24c8013df6412c755088853f531b1ffb5f20  ./HttpCommand/pom.xml
 8180c8a378dbbbecf09271a55bdf72fbe63c55c2  ./pom.xml
 6b1c1bbf94ffbf4124a58dede18d3916bad4b9ad  ./security-client-agnostic/pom.xml
-ec6efd34af68e1910237e5f9b2b67bd20aabcea1  ./timbuctoo-instancev4/pom.xml
+ce0256285638e52833811467ee366f7e6eb21363  ./timbuctoo-instancev4/pom.xml
 d9c324fb24b3c7a11fc57cc2bb324fb4449a1e7f  ./timbuctoo-test-services/pom.xml
 021b25dc5b52fe7cd04089e7cde13d950c188add  ./devtools/update_dependencies/dependencyparser.js

third_party/generate_workspace.bzl

@@ -390,7 +390,7 @@ def generated_maven_jars():
   )
   native.maven_jar(
     name = "commons_codec_commons_codec",
-    artifact = "commons-codec:commons-codec:jar:1.9",
+    artifact = "commons-codec:commons-codec:jar:1.10",
   )
   native.maven_jar(
     name = "io_dropwizard_metrics_metrics_httpclient",
@@ -649,6 +649,14 @@ def generated_maven_jars():
     name = "com_healthmarketscience_jackcess_jackcess",
     artifact = "com.healthmarketscience.jackcess:jackcess:jar:2.1.6",
   )
+  native.maven_jar(
+    name = "org_elasticsearch_client_rest",
+    artifact = "org.elasticsearch.client:rest:jar:5.5.3",
+  )
+  native.maven_jar(
+    name = "org_apache_httpcomponents_httpasyncclient",
+    artifact = "org.apache.httpcomponents:httpasyncclient:jar:4.1.2",
+  )
   native.maven_jar(
     name = "org_jsoup_jsoup",
     artifact = "org.jsoup:jsoup:jar:1.9.2",
@@ -2617,6 +2625,26 @@ def generated_java_libraries():
       ":commons_logging_commons_logging",
     ],
   )
+  native.java_library(
+    name = "org_elasticsearch_client_rest",
+    visibility = ["//visibility:public"],
+    exports = ["@org_elasticsearch_client_rest//jar"],
+    runtime_deps = [
+      ":org_apache_httpcomponents_httpclient",
+      ":org_apache_httpcomponents_httpcore",
+      ":org_apache_httpcomponents_httpasyncclient",
+      ":org_apache_httpcomponents_httpcore_nio",
+      ":commons_codec_commons_codec",
+      ":commons_logging_commons_logging",
+    ],
+  )
+  native.java_library(
+    name = "org_apache_httpcomponents_httpasyncclient",
+    visibility = ["//visibility:public"],
+    exports = ["@org_apache_httpcomponents_httpasyncclient//jar"],
+    runtime_deps = [
+    ],
+  )
   native.java_library(
     name = "org_jsoup_jsoup",
     visibility = ["//visibility:public"],

timbuctoo-instancev4/docker_config.yaml

@@ -18,7 +18,14 @@ securityConfiguration:
     authenticationCredentials: DUMMY
 
 baseUri: ${BASE_URI} # used to generate next and previous links of a search result
-userRedirectUrl: ${TIMBUCTOO_GUI_PUBLIC_URL}
+userRedirectUrl: ${BASE_URI}/static/graphiql
+
+elasticSearch:
+  hostname: ${TIMBUCTOO_ELASTICSEARCH_HOST}
+  port: ${TIMBUCTOO_ELASTICSEARCH_PORT}
+  username: ${TIMBUCTOO_ELASTICSEARCH_USER}
+  password: ${TIMBUCTOO_ELASTICSEARCH_PASSWORD}
+
 
 archetypesSchema: |
   type Archetypes {

timbuctoo-instancev4/example_config.yaml

@@ -16,7 +16,13 @@ securityConfiguration:
     authenticationCredentials: DUMMY
 
 baseUri: http://127.0.0.1:0 #port will be replaced with the port that timbuctoo ends up listening on
-userRedirectUrl: http://example.com
+userRedirectUrl: ./static/graphiql
+
+elasticSearch:
+  hostname: http://example.com/elasticsearchinstance
+  port: 80
+  username: foo
+  password: bar
 
 archetypesSchema: |
   type Archetypes {

timbuctoo-instancev4/pom.xml

@@ -302,6 +302,14 @@
       <version>2.1.6</version>
     </dependency>
 
+    <!-- ### Search API's ## -->
+    <!-- ElasticSearch REST API -->
+    <dependency>
+      <groupId>org.elasticsearch.client</groupId>
+      <artifactId>rest</artifactId>
+      <version>5.5.3</version>
+    </dependency>
+
     <!-- ### HTML Parser ## -->
     <dependency>
       <groupId>org.jsoup</groupId>

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooConfiguration.java

@@ -26,6 +26,7 @@
 import nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.DataStoreCreationException;
 import nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbDataStoreFactory;
 import nl.knaw.huygens.timbuctoo.v5.datastores.resourcesync.ResourceSync;
+import nl.knaw.huygens.timbuctoo.v5.elasticsearch.ElasticSearch;
 import nl.knaw.huygens.timbuctoo.v5.util.TimbuctooRdfIdHelper;
 import org.immutables.value.Value;
 
@@ -109,6 +110,8 @@ public HttpClientConfiguration getHttpClientConfiguration() {
   @Valid
   public abstract DataSetConfiguration getDataSetConfiguration();
 
+  public abstract ElasticSearch getElasticSearch();
+
   @JsonIgnore
   public DataSetRepository getDataSet() throws DataStoreCreationException {
     try {

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/server/TimbuctooV4.java

@@ -91,11 +91,11 @@
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.CreateDataSet;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.DataSet;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.ErrorResponseHelper;
+import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.GraphQl;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.JsonLdEditEndpoint;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.RdfUpload;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.ResourceSyncEndpoint;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.Rml;
-import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.GraphQl;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.TabularUpload;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.endpoints.WellKnown;
 import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.PaginationArgumentsHelper;
@@ -281,8 +281,9 @@ public void run(TimbuctooConfiguration configuration, Environment environment) t
         dataSetRepository,
         serializerWriterRegistry,
         configuration.getArchetypesSchema(),
-        new RdfWiringFactory(dataSetRepository),
-        new DerivedSchemaTypeGenerator(new PaginationArgumentsHelper())
+        new RdfWiringFactory(dataSetRepository, configuration.getElasticSearch()),
+        new DerivedSchemaTypeGenerator(new PaginationArgumentsHelper()),
+        environment.getObjectMapper()
       ),
       serializerWriterRegistry,
       securityConfig.getLoggedInUsers()

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/BUILD

@@ -27,6 +27,10 @@ java_library(
       "//timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/security",
       "//third_party:com_github_jsonld_java_jsonld_java",
       "//third_party:org_apache_httpcomponents_httpclient",
+      "//third_party:org_apache_httpcomponents_httpcore",
+      "//third_party:org_apache_httpcomponents_httpcore_nio",
+      "//third_party:org_apache_httpcomponents_httpasyncclient",
+      "//third_party:org_elasticsearch_client_rest",
       "//third_party:org_apache_jena_jena_core",
       "//third_party:com_google_code_findbugs_jsr305",
       "//third_party:org_apache_commons_commons_csv",

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/DataSetRepository.java

@@ -111,6 +111,12 @@ public Optional<DataSet> getDataSet(String ownerId, String dataSetId) {
     }
   }
 
+  public Optional<DataSet> getDataSet(String combinedId) {
+    final Tuple<String, String> splitId = PromotedDataSet.splitCombinedId(combinedId);
+    return Optional.ofNullable(dataSetMap.get(splitId.getLeft()))
+      .map(userDataSets -> userDataSets.get(splitId.getRight()));
+  }
+
   public DataSet createDataSet(String ownerId, String dataSetId) throws DataStoreCreationException {
     final PromotedDataSet dataSet = promotedDataSet(ownerId, dataSetId, rdfIdHelper.dataSet(ownerId, dataSetId), false);
     synchronized (dataSetMap) {
@@ -141,26 +147,18 @@ public boolean dataSetExists(String ownerId, String dataSet) {
     return getDataSet(ownerId, dataSet).isPresent();
   }
 
-  public Map<String, Set<PromotedDataSet>> getDataSets() {
-    return storedDataSets.getData();
+  public Collection<DataSet> getDataSets() {
+    return dataSetMap.values().stream().flatMap(x -> x.values().stream()).collect(Collectors.toList());
   }
 
-  public Map<String, Set<PromotedDataSet>> getPromotedDataSets() {
-    Map<String, Set<PromotedDataSet>> dataSets = storedDataSets.getData();
-    Map<String, Set<PromotedDataSet>> promotedDataSets = new HashMap<>();
-
-    for (Map.Entry<String, Set<PromotedDataSet>> userDataSets : dataSets.entrySet()) {
-      Set<PromotedDataSet> mappedUserSets = userDataSets.getValue()
-                                                        .stream()
-                                                        .filter(dataSet -> dataSet.isPromoted())
-                                                        .collect(Collectors.toSet());
-      promotedDataSets.put(userDataSets.getKey(), mappedUserSets);
-    }
-    return promotedDataSets;
+  public Collection<DataSet> getPromotedDataSets() {
+    return dataSetMap.values().stream().flatMap(x -> x.values().stream())
+      .filter(x -> x.getMetadata().isPromoted())
+      .collect(Collectors.toList());
   }
 
-  public Collection<PromotedDataSet> getDataSetsWithWriteAccess(String userId) {
-    List<PromotedDataSet> dataSetsWithWriteAccess = new ArrayList<>();
+  public Collection<DataSet> getDataSetsWithWriteAccess(String userId) {
+    List<DataSet> dataSetsWithWriteAccess = new ArrayList<>();
 
     for (Map<String, DataSet> userDataSets : dataSetMap.values()) {
       for (DataSet dataSet : userDataSets.values()) {
@@ -170,7 +168,7 @@ public Collection<PromotedDataSet> getDataSetsWithWriteAccess(String userId) {
             .map(VreAuthorization::isAllowedToWrite)
             .orElse(false);
           if (isAllowedToWrite) {
-            dataSetsWithWriteAccess.add(dataSet.getMetadata());
+            dataSetsWithWriteAccess.add(dataSet);
           }
         } catch (AuthorizationUnavailableException e) {
           LOG.error("Could not fetch authorization", e);

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dataset/dto/PromotedDataSet.java

@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import nl.knaw.huygens.timbuctoo.util.Tuple;
 import org.immutables.value.Value;
 
 import java.util.Optional;
@@ -26,6 +27,11 @@ public interface PromotedDataSet {
 
   Optional<String> role = Optional.empty();
 
+  static Tuple<String, String> splitCombinedId(String combinedId) {
+    String[] parts = combinedId.split("__", 2);
+    return Tuple.tuple(parts[0], parts[1]);
+  }
+
   static PromotedDataSet promotedDataSet(String ownerId, String dataSetId, String baseUri, boolean promoted) {
     if (!ownerId.matches(VALID_ID) || !dataSetId.matches(VALID_ID)) {
       throw new IllegalArgumentException("Owner id and dataSet id should match " + VALID_ID);

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/implementations/json/JsonSchemaStore.java

@@ -2,15 +2,16 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import nl.knaw.huygens.timbuctoo.v5.dataset.DataProvider;
-import nl.knaw.huygens.timbuctoo.v5.jsonfilebackeddata.JsonFileBackedData;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaEntityProcessor;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaStore;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaUpdateException;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaUpdater;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Type;
+import nl.knaw.huygens.timbuctoo.v5.jsonfilebackeddata.JsonFileBackedData;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 
 public class JsonSchemaStore implements SchemaStore {
@@ -38,7 +39,12 @@ public JsonSchemaStore(DataProvider dataProvider, File schemaLocation)
 
   @Override
   public Map<String, Type> getTypes() {
-    return schemaFile.getData();
+    final Map<String, Type> data = schemaFile.getData();
+    if (data == null) {
+      return Collections.emptyMap();
+    } else {
+      return data;
+    }
   }
 
   @Override

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/datastores/schemastore/SchemaEntityProcessor.java

@@ -1,10 +1,11 @@
 package nl.knaw.huygens.timbuctoo.v5.datastores.schemastore;
 
 import com.google.common.collect.ListMultimap;
-import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction;
 import nl.knaw.huygens.timbuctoo.v5.dataset.EntityProcessor;
-import nl.knaw.huygens.timbuctoo.v5.dataset.dto.PredicateData;
 import nl.knaw.huygens.timbuctoo.v5.dataset.PredicateHandler;
+import nl.knaw.huygens.timbuctoo.v5.dataset.dto.PredicateData;
+import nl.knaw.huygens.timbuctoo.v5.dataset.dto.RelationPredicate;
+import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Predicate;
 import nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Type;
 import nl.knaw.huygens.timbuctoo.v5.util.RdfConstants;
@@ -45,7 +46,7 @@ public void processEntity(String cursor, String subjectUri, ListMultimap<String,
     //Step 1: create all the types that this subject belongs to
     Map<String, Type> curTypes = new HashMap<>();
     List<PredicateData> subjectTypes = addedPredicates.get(RDF_TYPE);
-    if (subjectTypes.isEmpty()) {
+    if (subjectTypes.stream().noneMatch(p -> p instanceof RelationPredicate)) {
       curTypes.put(UNKNOWN, types.computeIfAbsent(UNKNOWN, TYPE_MAKER));
     } else {
       for (PredicateData type : subjectTypes) {

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/GraphQl.java

@@ -11,7 +11,7 @@
 import nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.contenttypes.SerializerWriter;
 import nl.knaw.huygens.timbuctoo.v5.dropwizard.contenttypes.SerializerWriterRegistry;
-import nl.knaw.huygens.timbuctoo.v5.graphql.rootquery.RootQuery;
+import nl.knaw.huygens.timbuctoo.v5.graphql.datafetchers.dto.RootData;
 import nl.knaw.huygens.timbuctoo.v5.graphql.serializable.SerializerExecutionStrategy;
 import nl.knaw.huygens.timbuctoo.v5.serializable.SerializableResult;
 
@@ -147,7 +147,7 @@ public Response executeGraphql(String query, String acceptHeader, String acceptP
     }
     final ExecutionResult result = graphQl
       .execute(newExecutionInput()
-        .root(new RootQuery.RootData(loggedInUsers.userFor(authHeader)))
+        .root(new RootData(loggedInUsers.userFor(authHeader)))
         .query(queryFromBody)
         .operationName(operationName)
         .variables(variables == null ? Collections.emptyMap() : variables)

timbuctoo-instancev4/src/main/java/nl/knaw/huygens/timbuctoo/v5/dropwizard/endpoints/WellKnown.java

@@ -10,7 +10,7 @@ public class WellKnown {
 
   @Path("resourcesync")
   @GET
-  public Response resoureSync() {
+  public Response resourceSync() {
     // Permanent redirect
     return Response.seeOther(UriBuilder.fromResource(ResourceSyncEndpoint.class)
                                        .path(ResourceSyncEndpoint.SOURCE_DESCRIPTION_PATH)