diff --git a/karma-commands/commands-publish-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/alignment/GenerateR2RMLModelCommand.java b/karma-commands/commands-publish-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/alignment/GenerateR2RMLModelCommand.java index f25b4705c..2ad565b4f 100644 --- a/karma-commands/commands-publish-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/alignment/GenerateR2RMLModelCommand.java +++ b/karma-commands/commands-publish-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/alignment/GenerateR2RMLModelCommand.java @@ -278,6 +278,18 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { // ***************************************************************************************** // ***************************************************************************************** + + // Check if the flag is set to enable semantic labeling. + // If yes then upload semantic types and columns to the server + Boolean onlineSemanticTypingEnabled = modelingConfiguration.getOnlineSemanticTypingEnabled(); + if(onlineSemanticTypingEnabled) { + try { + semanticModel.uploadUserSemanticTypes(); + } catch (Exception e) { + logger.error("Unable to upload semantic types"); + } + } + try { R2RMLAlignmentFileSaver fileSaver = new R2RMLAlignmentFileSaver(workspace); diff --git a/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/SetKarmaClientNameCommand.java b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/SetKarmaClientNameCommand.java new file mode 100644 index 000000000..8601c452b --- /dev/null +++ b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/SetKarmaClientNameCommand.java @@ -0,0 +1,110 @@ +package edu.isi.karma.controller.command; + +import edu.isi.karma.config.ModelingConfigurationRegistry; +import edu.isi.karma.controller.update.AbstractUpdate; +import edu.isi.karma.controller.update.ErrorUpdate; +import edu.isi.karma.controller.update.UpdateContainer; +import edu.isi.karma.rep.Workspace; +import edu.isi.karma.view.VWorkspace; +import edu.isi.karma.webserver.ContextParametersRegistry; +import edu.isi.karma.webserver.ServletContextParameterMap; +import org.json.JSONStringer; +import org.json.JSONWriter; +import org.slf4j.LoggerFactory; + +import org.slf4j.Logger; + +import java.io.*; +import java.util.Properties; + +/** + * Created by alse on 10/4/16. + * Every karma client is going to be assigned a random string as an id. + * This id - karma client name - can be changed to a given value by calling this command + */ +public class SetKarmaClientNameCommand extends Command { + private String karmaClientName; + private String property = "karma.client.name"; + + private static Logger logger = LoggerFactory.getLogger(SetKarmaClientNameCommand.class); + + protected SetKarmaClientNameCommand(String id, String model, String name) { + super(id, model); + karmaClientName = name; + } + + @Override + public String getCommandName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getTitle() { + return "Set Karma name Configuration"; + } + + @Override + public String getDescription() { + return null; + } + + @Override + public CommandType getCommandType() { + return CommandType.notInHistory; + } + + @Override + public UpdateContainer doIt(Workspace workspace) throws CommandException { + UpdateContainer uc = new UpdateContainer(); + try{ + + uc.add(new AbstractUpdate() { + @Override + public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) { + try { + // read modeling.properties file + String fileName = ContextParametersRegistry.getInstance() + .getContextParameters(ContextParametersRegistry.getInstance().getDefault().getId()) + .getParameterValue(ServletContextParameterMap.ContextParameter.USER_CONFIG_DIRECTORY) + + "/modeling.properties"; + + BufferedReader file = new BufferedReader(new FileReader(fileName)); + String line; + String modelingPropertiesContent = ""; + while ((line = file.readLine()) != null) { + if (line.startsWith(property)) { + modelingPropertiesContent += property + "=" + karmaClientName + '\n'; + } else { + modelingPropertiesContent += line + '\n'; + } + } + file.close(); + FileOutputStream fileOut = new FileOutputStream(fileName); + fileOut.write(modelingPropertiesContent.getBytes()); + fileOut.close(); + + // reload modeling configuration + ModelingConfigurationRegistry.getInstance().getModelingConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).load(); + JSONStringer jsonStr = new JSONStringer(); + + JSONWriter writer = jsonStr.object(); + writer.key("updateType").value(this.getClass().getName()); + writer.endObject(); + pw.print(writer.toString()); + } catch (Exception e) { + logger.error("Error updating Modeling Configuraion", e); + } + } + }); + } catch (Exception e) { + logger.error("Error updating Modeling Configuraion:" , e); + uc.add(new ErrorUpdate("Error updating Modeling Configuraion")); + } + return uc; + } + + @Override + public UpdateContainer undoIt(Workspace workspace) { + return null; + } +} diff --git a/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/SetKarmaClientNameCommandFactory.java b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/SetKarmaClientNameCommandFactory.java new file mode 100644 index 000000000..a0a212bdc --- /dev/null +++ b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/SetKarmaClientNameCommandFactory.java @@ -0,0 +1,22 @@ +package edu.isi.karma.controller.command; +import edu.isi.karma.rep.Workspace; + +import javax.servlet.http.HttpServletRequest; + +/** + * Created by alse on 10/4/16. + */ +public class SetKarmaClientNameCommandFactory extends CommandFactory{ + enum Arguments { + value + } + @Override + public Command createCommand(HttpServletRequest request, Workspace workspace) { + return new SetKarmaClientNameCommand(getNewId(workspace), Command.NEW_MODEL, request.getParameter(Arguments.value.name())); + } + + @Override + public Class getCorrespondingCommand() { + return SetKarmaClientNameCommand.class; + } +} diff --git a/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/ToggleOnlineSemanticTypingCommand.java b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/ToggleOnlineSemanticTypingCommand.java new file mode 100644 index 000000000..d0fb8ccd1 --- /dev/null +++ b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/ToggleOnlineSemanticTypingCommand.java @@ -0,0 +1,125 @@ +package edu.isi.karma.controller.command; + +import com.sun.org.apache.xpath.internal.operations.Bool; +import edu.isi.karma.config.ModelingConfigurationRegistry; +import edu.isi.karma.controller.update.AbstractUpdate; +import edu.isi.karma.controller.update.ErrorUpdate; +import edu.isi.karma.controller.update.UpdateContainer; +import edu.isi.karma.rep.Workspace; +import edu.isi.karma.semantictypes.typinghandler.HybridSTModelHandler; +import edu.isi.karma.semantictypes.typinghandler.RemoteSTModelHandler; +import edu.isi.karma.view.VWorkspace; +import edu.isi.karma.webserver.ContextParametersRegistry; +import edu.isi.karma.webserver.ServletContextParameterMap; +import org.json.JSONStringer; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.Properties; + +/** + * Created by alse on 9/29/16. + * This function toggles the setting in modeling.properties file responsible for setting the semantictyper to local or remote + * Based on the current value of the property, this just toggles the value + */ +public class ToggleOnlineSemanticTypingCommand extends Command { + + private String property = "online.semantic.typing"; + + private static Logger logger = LoggerFactory.getLogger(UpdateUIConfigurationCommand.class); + + protected ToggleOnlineSemanticTypingCommand(String id, String model) { + super(id, model); + } + + @Override + public String getCommandName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getTitle() { + return "Set Modeling Configuration"; + } + + @Override + public String getDescription() { + return null; + } + + @Override + public CommandType getCommandType() { + return CommandType.notInHistory; + } + + @Override + public UpdateContainer doIt(Workspace workspace) throws CommandException { + UpdateContainer uc = new UpdateContainer(); + try{ + + uc.add(new AbstractUpdate() { + @Override + public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) { + try { + // read modeling.properties file + String fileName = ContextParametersRegistry.getInstance() + .getContextParameters(ContextParametersRegistry.getInstance().getDefault().getId()) + .getParameterValue(ServletContextParameterMap.ContextParameter.USER_CONFIG_DIRECTORY) + + "/modeling.properties"; + + Properties prop = new Properties(); + prop.load(new FileInputStream(fileName)); + + BufferedReader file = new BufferedReader(new FileReader(fileName)); + String line; + String modelingPropertiesContent = ""; + while ((line = file.readLine()) != null) { + if (line.startsWith(property)) { + String value = Boolean.toString(!Boolean.valueOf(prop.getProperty(property))); + modelingPropertiesContent += property + "=" + value + '\n'; + } else { + modelingPropertiesContent += line + '\n'; + } + } + file.close(); + FileOutputStream fileOut = new FileOutputStream(fileName); + fileOut.write(modelingPropertiesContent.getBytes()); + fileOut.close(); + + String contextId = vWorkspace.getWorkspace().getContextId(); + Boolean isModelEnabled = vWorkspace.getWorkspace().getSemanticTypeModelHandler().getModelHandlerEnabled(); + + // If online is enabled then use RemoteSTModelHandler for semantic typing else use HybridSTModelHandler + if (!Boolean.valueOf(prop.getProperty(property))){ + vWorkspace.getWorkspace().setSemanticTypeModelHandler(new RemoteSTModelHandler(contextId)); + } else { + vWorkspace.getWorkspace().setSemanticTypeModelHandler(new HybridSTModelHandler(contextId)); + } + vWorkspace.getWorkspace().getSemanticTypeModelHandler().setModelHandlerEnabled(isModelEnabled); + + ModelingConfigurationRegistry.getInstance().getModelingConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).load(); + JSONStringer jsonStr = new JSONStringer(); + + JSONWriter writer = jsonStr.object(); + writer.key("updateType").value(this.getClass().getName()); + writer.endObject(); + pw.print(writer.toString()); + } catch (Exception e) { + logger.error("Error updating Modeling Configuraion", e); + } + } + }); + } catch (Exception e) { + logger.error("Error updating Modeling Configuraion:" , e); + uc.add(new ErrorUpdate("Error updating Modeling Configuraion")); + } + return uc; + } + + @Override + public UpdateContainer undoIt(Workspace workspace) { + return null; + } +} diff --git a/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/ToggleOnlineSemanticTypingCommandFactory.java b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/ToggleOnlineSemanticTypingCommandFactory.java new file mode 100644 index 000000000..73e709f91 --- /dev/null +++ b/karma-commands/commands-update/src/main/java/edu/isi/karma/controller/command/ToggleOnlineSemanticTypingCommandFactory.java @@ -0,0 +1,22 @@ +package edu.isi.karma.controller.command; + +import edu.isi.karma.rep.Workspace; + +import javax.servlet.http.HttpServletRequest; + +/** + * Created by alse on 9/29/16. + * Request @param property: is the name of property in modeling.properties eg. online.semantic.typing, train.on.apply.history + * Request @param value: is the value of the property to be set. + */ +public class ToggleOnlineSemanticTypingCommandFactory extends CommandFactory{ + @Override + public Command createCommand(HttpServletRequest request, Workspace workspace) { + return new ToggleOnlineSemanticTypingCommand(getNewId(workspace), Command.NEW_MODEL); + } + + @Override + public Class getCorrespondingCommand() { + return ToggleOnlineSemanticTypingCommand.class; + } +} diff --git a/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java b/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java index 3de248bcd..07ab94fb2 100644 --- a/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java +++ b/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java @@ -30,6 +30,7 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.Properties; +import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +82,9 @@ public class ModelingConfiguration { private Boolean showModelsWithoutMatching; private String defaultProperty = null; - + private Boolean onlineSemanticTypingEnabled; // decides whether we should use remote semantic typing or local + private String karmaClientName; // name of the instance of karma set to a random uuid string by default + private final String newLine = System.getProperty("line.separator"); private String defaultModelingProperties = @@ -92,7 +95,8 @@ public class ModelingConfiguration { "##########################################################################################" + newLine + "" + newLine + "train.on.apply.history=false" + newLine + - "predict.on.apply.history=false" + newLine + + "predict.on.apply.history=false" + newLine + + "online.semantic.typing=false" + newLine + "" + newLine + "##########################################################################################" + newLine + "#" + newLine + @@ -163,10 +167,13 @@ public class ModelingConfiguration { "##########################################################################################" + newLine + "" + newLine + "models.display.nomatching=false" + newLine + - "history.store.old=false" + "history.store.old=false" + + "#" + newLine + + "##########################################################################################" + newLine + + "" + newLine + + "karma.client.name=" + UUID.randomUUID().toString() ; - public void load() { try { Properties modelingProperties = loadParams(); @@ -290,6 +297,11 @@ public void load() { out.println("default.property=http://schema.org/name"); out.close(); } + + // set it to false by default + onlineSemanticTypingEnabled = Boolean.parseBoolean(modelingProperties.getProperty("online.semantic.typing", "false")); + // random uuid is set by default + karmaClientName = modelingProperties.getProperty("karma.client.name", UUID.randomUUID().toString()); } catch (IOException e) { logger.error("Error occured while reading config file ...", e); System.exit(1); @@ -543,7 +555,19 @@ public String getDefaultProperty() { load(); return defaultProperty; } - + + public Boolean getOnlineSemanticTypingEnabled(){ + if(onlineSemanticTypingEnabled == null) + load(); + return onlineSemanticTypingEnabled; + } + + public String getKarmaClientName(){ + if(karmaClientName == null) + load(); + return karmaClientName; + } + public void setManualAlignment() { ontologyAlignment = false; diff --git a/karma-common/src/main/java/edu/isi/karma/modeling/alignment/SemanticModel.java b/karma-common/src/main/java/edu/isi/karma/modeling/alignment/SemanticModel.java index 7faab8527..04a9817a1 100644 --- a/karma-common/src/main/java/edu/isi/karma/modeling/alignment/SemanticModel.java +++ b/karma-common/src/main/java/edu/isi/karma/modeling/alignment/SemanticModel.java @@ -21,12 +21,8 @@ package edu.isi.karma.modeling.alignment; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; +import java.io.*; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -37,6 +33,11 @@ import java.util.Map.Entry; import java.util.Set; +import edu.isi.karma.config.ModelingConfiguration; +import edu.isi.karma.config.ModelingConfigurationRegistry; +import edu.isi.karma.rep.Row; +import edu.isi.karma.semantictypes.remote.SemanticLabelingService; +import edu.isi.karma.webserver.WorkspaceKarmaHomeRegistry; import org.jgrapht.graph.DirectedWeightedMultigraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -256,7 +257,75 @@ private void setUserTypesForColumnNodes() { logger.debug("The column node " + ((ColumnNode)n).getColumnName() + " does not have any domain or it has more than one domain."); } } - + // This function uploads the semantic types to the server and then uploads the columns by calling uploadUserColumn + public void uploadUserSemanticTypes() { + Map semanticTypes = this.worksheet.getSemanticTypes().getTypes(); + Map> columns = new HashMap<>(); + // get all the columns (max 1000 rows) + for(Row row: this.worksheet.getDataTable().getRows(0, 1000, new SuperSelection("DEFAULT") )){ + for(Entry node: row.getNodesMap().entrySet()){ + if (columns.containsKey(node.getKey())) { + columns.get(node.getKey()).add(node.getValue().getValue().asString()); + } else { + List values = new ArrayList<>(); + values.add(node.getValue().getValue().asString()); + columns.put(node.getKey(), values); + } + } + } + + ModelingConfiguration modelingConfiguration = ModelingConfigurationRegistry.getInstance().getModelingConfiguration(WorkspaceKarmaHomeRegistry.getInstance().getKarmaHome(workspace.getId())); + // This is the name of the model (not title of the worksheet) + String modelName = SemanticLabelingService.getModelName(this.worksheet.getMetadataContainer().getWorksheetProperties().getJSONRepresentation().get("graphLabel").toString(), modelingConfiguration.getKarmaClientName()); + + // Try deleting all the existing columns based on the model name + // If it returns 404 it means that it is not there to begin with. So we can simply ignore it. Print the stacktrace anyways + try { + new SemanticLabelingService().deleteModel(URLEncoder.encode(modelName, "UTF-8")); + } catch (IOException e) { + e.printStackTrace(); + } + + for(Map.Entry> column: columns.entrySet()){ + String domain = semanticTypes.get(column.getKey()).getDomain().getUri(); + String type = semanticTypes.get(column.getKey()).getType().getUri(); + // create semantic type on the server + try { + String query = "class=" + URLEncoder.encode(domain , "UTF-8") + "&property=" + URLEncoder.encode(type, "UTF-8"); + new SemanticLabelingService().post(query); + } catch (Exception e) { + logger.warn("Semantic type domain:"+ domain +" type:"+ type +" can't be created."); + e.printStackTrace(); + } + // add column data to the server + uploadUserColumn(domain, type, column); + } + } + // This function uploads the columns to the semantic typer service + private void uploadUserColumn(String domain, String type, Map.Entry> column){ + String data = ""; + for(String value : column.getValue()){ + data += value+"\n"; + } + ModelingConfiguration modelingConfiguration = ModelingConfigurationRegistry.getInstance().getModelingConfiguration(WorkspaceKarmaHomeRegistry.getInstance().getKarmaHome(workspace.getId())); + String modelName = SemanticLabelingService.getModelName(this.worksheet.getMetadataContainer().getWorksheetProperties().getJSONRepresentation().get("graphLabel").toString(), modelingConfiguration.getKarmaClientName()); + + // add all the columns + // note that karma client name is prepended to all the column names. + // We don't send plain column names to the server - we need this to differentiate it from rest of the users + try { + String id = SemanticLabelingService.getSemanticTypeId(domain, type); + String query = "columnName=" + URLEncoder.encode( + SemanticLabelingService.getColumnName(this.worksheet.getHeaders().getHNode(column.getKey()).getColumnName(), modelingConfiguration.getKarmaClientName()), + "UTF-8") + + "&sourceName="+ URLEncoder.encode(this.worksheet.getTitle(), "UTF-8") + + "&model="+URLEncoder.encode(modelName, "UTF-8"); + new SemanticLabelingService().post(query, id, data); + } catch (Exception e) { + logger.warn("Can't upload column:"+ this.worksheet.getHeaders().getHNode(column.getKey()).getColumnName() +" domain:"+ domain +" type:"+ type +" can't be created."); + e.printStackTrace(); + } + } public void print() { logger.info("id: " + this.getId()); logger.info("name: " + this.getName()); diff --git a/karma-common/src/main/java/edu/isi/karma/modeling/semantictypes/SemanticTypeUtil.java b/karma-common/src/main/java/edu/isi/karma/modeling/semantictypes/SemanticTypeUtil.java index 9ffd8250e..8c4b2d469 100644 --- a/karma-common/src/main/java/edu/isi/karma/modeling/semantictypes/SemanticTypeUtil.java +++ b/karma-common/src/main/java/edu/isi/karma/modeling/semantictypes/SemanticTypeUtil.java @@ -20,17 +20,11 @@ ******************************************************************************/ package edu.isi.karma.modeling.semantictypes; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; +import java.util.*; import java.util.Map.Entry; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.Condition; -import java.util.Random; -import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -222,7 +216,15 @@ public SemanticTypeColumnModel predictColumnSemanticType(Workspace workspace, Wo ISemanticTypeModelHandler modelHandler = workspace.getSemanticTypeModelHandler(); OntologyManager ontologyManager = workspace.getOntologyManager(); - + + Set namespaces = new HashSet<>(); + for (Label label : ontologyManager.getClasses().values()) { + namespaces.add(label.getNs()); + } + // set namespaces in model handler + // this is mostly needed in remote semantic typer where we filter predictions based on preloaded ontologies + modelHandler.setNamespaces(new ArrayList<>(namespaces)); + List result = modelHandler.predictType(trainingExamples, numSuggestions); if (result == null) { logger.debug("Error occured while predicting semantic type."); @@ -391,6 +393,7 @@ public static void identifyOutliers(Worksheet worksheet, String predictedType, H logger.error("Error while predicting type for " + nodeVal); continue; } + // Check here if it is an outlier // System.out.println("Example: " + examples.get(0) + " Label: " + predictedLabels + " Score: " + confidenceScores); String predictedLabel = result.get(0).getLabel(); diff --git a/karma-common/src/main/java/edu/isi/karma/rep/Workspace.java b/karma-common/src/main/java/edu/isi/karma/rep/Workspace.java index 00ed04f93..e85e87a7b 100644 --- a/karma-common/src/main/java/edu/isi/karma/rep/Workspace.java +++ b/karma-common/src/main/java/edu/isi/karma/rep/Workspace.java @@ -25,6 +25,10 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import edu.isi.karma.config.ModelingConfiguration; +import edu.isi.karma.config.ModelingConfigurationRegistry; +import edu.isi.karma.semantictypes.typinghandler.RemoteSTModelHandler; +import edu.isi.karma.webserver.WorkspaceKarmaHomeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +79,7 @@ public class Workspace extends Entity { /** * The CRF Model for the workspace */ - private final ISemanticTypeModelHandler semTypeModelHandler; + private ISemanticTypeModelHandler semTypeModelHandler; private final CommandPreferences commandPreferences; @@ -151,6 +155,10 @@ public ISemanticTypeModelHandler getSemanticTypeModelHandler() { return semTypeModelHandler; } + public void setSemanticTypeModelHandler(ISemanticTypeModelHandler semanticTypeModelHandler) { + this.semTypeModelHandler = semanticTypeModelHandler; + } + public CommandPreferences getCommandPreferences() { return commandPreferences; } diff --git a/karma-typer/src/main/java/edu/isi/karma/semantictypes/remote/SemanticLabelingService.java b/karma-typer/src/main/java/edu/isi/karma/semantictypes/remote/SemanticLabelingService.java new file mode 100644 index 000000000..f76211198 --- /dev/null +++ b/karma-typer/src/main/java/edu/isi/karma/semantictypes/remote/SemanticLabelingService.java @@ -0,0 +1,177 @@ +package edu.isi.karma.semantictypes.remote; + +import edu.isi.karma.modeling.semantictypes.SemanticTypeLabel; +import edu.isi.karma.modeling.semantictypes.SemanticTypeLabelComparator; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.StringUtils; +import org.json.JSONArray; +import sun.net.www.protocol.http.HttpURLConnection; + +import java.io.*; +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.StringJoiner; + +/** + * Created by alse on 10/4/16. + * This is the class interacting with the semantic typer service + * Components need to call functions in this to interact with the service and not directly + */ +public class SemanticLabelingService { + private static String SEMANTIC_TYPE_PART = "/semantic_types"; + private static String COLUMN_PART = "/type"; + private static String PREDICT = "/predict"; + // This is the place where the service is hosted + // Should we map this to some domain so that we dont have to keep making changes in the code to move the service to a different machine? + private static String BASE_URL = "http://52.38.65.60:80"; + private static String ID_DIVIDER = "-"; // The divider that is used to separate the different parts of ID's, like domain and type + private static String DEFAULT_MODEL = "default"; + + public SemanticLabelingService() { + + } + + // GET + private String GET(String urlPart) throws IOException { + URL url = new URL( BASE_URL + urlPart); + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setRequestMethod("GET"); + InputStream response = new BufferedInputStream(connection.getInputStream()); + BufferedReader reader = new BufferedReader(new InputStreamReader(response)); + StringBuilder result = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + result.append(line); + } + return result.toString(); + } + // POST + private String POST(String urlPart, String body) throws IOException { + URL url = new URL( BASE_URL + urlPart); + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setRequestMethod("POST"); + connection.setDoInput(true); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type","application/json"); + OutputStream os = connection.getOutputStream(); + os.write(body.getBytes("UTF-8")); + os.close(); + try { + InputStream response = new BufferedInputStream(connection.getInputStream()); + BufferedReader reader = new BufferedReader(new InputStreamReader(response)); + StringBuilder result = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + result.append(line); + } + return result.toString(); + } catch (IOException e){ + if (connection.getResponseCode() == java.net.HttpURLConnection.HTTP_CONFLICT) { + throw new IllegalStateException(connection.getResponseMessage()); + } else { + throw e; + } + } + } + // DELETE + private String DELETE(String urlPart) throws IOException { + URL url = new URL( BASE_URL + urlPart); + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setRequestMethod("DELETE"); + InputStream response = new BufferedInputStream(connection.getInputStream()); + BufferedReader reader = new BufferedReader(new InputStreamReader(response)); + StringBuilder result = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + result.append(line); + } + return result.toString(); + } + public String get(String query) throws IOException { + if (!query.equals("")) + query = "?" + query; + return GET(SEMANTIC_TYPE_PART + query); + } + public String post(String query) throws IOException { + if (!query.equals("")) + query = "?" + query; + return POST(SEMANTIC_TYPE_PART + query, ""); + } + // This is to delete column + public String delete(String id) throws IOException { + if (!id.equals("")) + id = "/" + id; + return DELETE(SEMANTIC_TYPE_PART + COLUMN_PART + id); + + } + // This deletes all columns for a given modelName + public void deleteModel(String modelName) throws IOException { + JSONArray semanticTypes = new JSONArray(get("models=" + modelName + "&returnColumns=true")); + for (int i = 0; i < semanticTypes.length(); i++) { + JSONArray columns = semanticTypes.getJSONObject(i).getJSONArray("columns"); + for (int j = 0; j < columns.length(); j++) { + try { + delete(columns.getJSONObject(j).getString("column_id")); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + public String get(String query, String id){ + return ""; + } + public String post(String query, String id, String body) throws IOException{ + if (!query.equals("")) + query = "?" + query; + return POST(SEMANTIC_TYPE_PART + "/" + id + query, body); + } + public String get(String query, String id, String column_id){ + return ""; + } + public List predict(String values, int numPredictions, List namespaces){ + List predictions = new ArrayList<>(); + try { + String query = ""; + if (namespaces != null && namespaces.size() > 0) { + StringJoiner sj = new StringJoiner(","); + for(String s: namespaces) { + if (!s.isEmpty() && (s.charAt(s.length()-1) == '#' || s.charAt(s.length()-1) == '/')) + s = s.substring(0,s.length()-1); + sj.add(s); + } + query = "?namespaces=" + URLEncoder.encode(sj.toString(), "UTF-8"); + } + JSONArray response = new JSONArray(POST(PREDICT + query, values)); + for (int i = 0; i < (response.length() < numPredictions ? response.length() : numPredictions); i++) + { + String [] label = response.getJSONObject(i).getString("type_id").split(ID_DIVIDER); + float score = (float) response.getJSONObject(i).getDouble("score"); + label[0] = StringUtils.newStringUtf8(Base64.decodeBase64(label[0])); + label[1] = StringUtils.newStringUtf8(Base64.decodeBase64(label[1])); + predictions.add(new SemanticTypeLabel(label[0] + "|" + label[1], score)); + } + Collections.sort(predictions, new SemanticTypeLabelComparator()); + } catch (IOException e) { + e.printStackTrace(); + } + + return predictions; + } + public static String getSemanticTypeId(String domain, String type) { + return Base64.encodeBase64String(domain.getBytes()) + ID_DIVIDER + Base64.encodeBase64String(type.getBytes()); + } + public static String getModelName(String modelName, String uuid) { + return uuid + ID_DIVIDER + modelName; + } + public static String getColumnName(String columnName, String uuid) { + return uuid + ID_DIVIDER + columnName; + } + // Note: column_name here should come from getColumnName + public static String getColumnId(String type_id, String column_name, String source_name) { + return type_id + ID_DIVIDER + Base64.encodeBase64String(column_name.getBytes()) + ID_DIVIDER + Base64.encodeBase64String(source_name.getBytes())+ ID_DIVIDER + Base64.encodeBase64String(DEFAULT_MODEL.getBytes()); + } +} diff --git a/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/HybridSTModelHandler.java b/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/HybridSTModelHandler.java index c8324cbfd..0062d9ca7 100644 --- a/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/HybridSTModelHandler.java +++ b/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/HybridSTModelHandler.java @@ -528,12 +528,22 @@ public boolean readModelFromFile(String filepath, boolean isNumeric) { return true; } + @Override + public void setNamespaces(List namespaces) { + + } + @Override public void setModelHandlerEnabled(boolean enabled) { this.modelEnabled = enabled; } + @Override + public boolean getModelHandlerEnabled() { + return this.modelEnabled; + } + @Override public boolean readModelFromFile(String filepath) { final ServletContextParameterMap contextParameters = ContextParametersRegistry.getInstance().getContextParameters(contextId); diff --git a/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/LuceneBasedSTModelHandler.java b/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/LuceneBasedSTModelHandler.java index d0abae695..1971c3172 100644 --- a/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/LuceneBasedSTModelHandler.java +++ b/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/LuceneBasedSTModelHandler.java @@ -365,10 +365,20 @@ public void setModelHandlerEnabled(boolean enabled) { } + @Override + public boolean getModelHandlerEnabled() { + return this.modelEnabled; + } + @Override public boolean readModelFromFile(String filepath, boolean isNumeric) { // TODO Auto-generated method stub return false; } + @Override + public void setNamespaces(List namespaces) { + + } + } diff --git a/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/RemoteSTModelHandler.java b/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/RemoteSTModelHandler.java new file mode 100644 index 000000000..e1a468351 --- /dev/null +++ b/karma-typer/src/main/java/edu/isi/karma/semantictypes/typinghandler/RemoteSTModelHandler.java @@ -0,0 +1,105 @@ +package edu.isi.karma.semantictypes.typinghandler; + +import edu.isi.karma.modeling.semantictypes.ISemanticTypeModelHandler; +import edu.isi.karma.modeling.semantictypes.SemanticTypeLabel; +import edu.isi.karma.semantictypes.remote.SemanticLabelingService; +import edu.isi.karma.webserver.ContextParametersRegistry; +import edu.isi.karma.webserver.ServletContextParameterMap; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by alse on 10/23/16. + * This class handles semantic typing using the service + */ +public class RemoteSTModelHandler implements ISemanticTypeModelHandler { + static Logger logger = LoggerFactory + .getLogger(HybridSTModelHandler.class.getSimpleName()); + + private String contextId; + private boolean modelEnabled = true; + private List namespaces; // list or URIs which limit the predictions to the ones loaded in this + + public RemoteSTModelHandler(String contextId) { + this.contextId = contextId; + } + @Override + public boolean addType(String label, List examples) { + return false; + } + + @Override + public List predictType(List examples, int numPredictions) { + if (!this.modelEnabled) { + logger.warn("Semantic Type Modeling is not enabled"); + return null; + } + // Sanity checks for arguments + if (examples == null || examples.isEmpty() || numPredictions <= 0) { + logger.warn("Invalid arguments. Possible problems: examples list size is zero, numPredictions is non-positive"); + return null; + } + + logger.debug("Predict Type for " + examples.toArray().toString()); + + logger.warn("-----------------------------------------------------------------------------"); + + StringBuilder sb = new StringBuilder(); + String sep = "\n"; + for(String s: examples) { + sb.append(sep).append(s); + } + + List predictions = new SemanticLabelingService().predict(sb.toString(), numPredictions, this.namespaces); + logger.debug("Got " + predictions.size() + " predictions"); + logger.warn("-----------------------------------------------------------------------------"); + return predictions; + } + + @Override + public boolean removeAllLabels() { + return false; + } + + @Override + public boolean readModelFromFile(String filepath, boolean isNumeric) { + final ServletContextParameterMap contextParameters = ContextParametersRegistry.getInstance().getContextParameters(contextId); + if (isNumeric) { + contextParameters + .setParameterValue(ServletContextParameterMap.ContextParameter.NUMERIC_SEMTYPE_MODEL_DIRECTORY, filepath); + } + else { + contextParameters + .setParameterValue(ServletContextParameterMap.ContextParameter.TEXTUAL_SEMTYPE_MODEL_DIRECTORY, filepath); + } + return true; + } + + @Override + public void setNamespaces(List namespaces) { + this.namespaces = new ArrayList<>(namespaces); + } + + @Override + public void setModelHandlerEnabled(boolean enabled) { + this.modelEnabled = enabled; + + } + + @Override + public boolean getModelHandlerEnabled() { + return this.modelEnabled; + } + + @Override + public boolean readModelFromFile(String filepath) { + final ServletContextParameterMap contextParameters = ContextParametersRegistry.getInstance().getContextParameters(contextId); + contextParameters + .setParameterValue(ServletContextParameterMap.ContextParameter.SEMTYPE_MODEL_DIRECTORY, filepath); + return true; + } +} diff --git a/karma-util/src/main/java/edu/isi/karma/modeling/semantictypes/ISemanticTypeModelHandler.java b/karma-util/src/main/java/edu/isi/karma/modeling/semantictypes/ISemanticTypeModelHandler.java index fa451d424..99e4113be 100644 --- a/karma-util/src/main/java/edu/isi/karma/modeling/semantictypes/ISemanticTypeModelHandler.java +++ b/karma-util/src/main/java/edu/isi/karma/modeling/semantictypes/ISemanticTypeModelHandler.java @@ -9,5 +9,7 @@ public interface ISemanticTypeModelHandler { public boolean removeAllLabels(); public boolean readModelFromFile(String filepath); public void setModelHandlerEnabled(boolean enabled); + public boolean getModelHandlerEnabled(); boolean readModelFromFile(String filepath, boolean isNumeric); + public void setNamespaces(List namespaces); } diff --git a/karma-web/src/main/java/edu/isi/karma/webserver/KarmaServlet.java b/karma-web/src/main/java/edu/isi/karma/webserver/KarmaServlet.java index 32f98d44a..3b06c29a0 100644 --- a/karma-web/src/main/java/edu/isi/karma/webserver/KarmaServlet.java +++ b/karma-web/src/main/java/edu/isi/karma/webserver/KarmaServlet.java @@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import edu.isi.karma.semantictypes.typinghandler.RemoteSTModelHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,7 +214,15 @@ public void generateJson(String prefix, PrintWriter pw, UIConfiguration uiConfiguration = UIConfigurationRegistry.getInstance().getUIConfiguration(vWorkspace.getWorkspace().getContextId()); uiConfiguration.loadConfig(); ModelingConfigurationRegistry.getInstance().register(vWorkspace.getWorkspace().getContextId()); - + + // If online semantic typer is enabled, then set model handler to edu.isi.karma.semantictypes.remote + Boolean isRemote = ModelingConfigurationRegistry.getInstance() + .getModelingConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()) + .getOnlineSemanticTypingEnabled(); + if (isRemote) { + vWorkspace.getWorkspace().setSemanticTypeModelHandler(new RemoteSTModelHandler(vWorkspace.getWorkspace().getContextId())); + } + //2 Return all settings related updates pw.println("{"); pw.println("\"updateType\": \"UISettings\", "); diff --git a/karma-web/src/main/webapp/index.jsp b/karma-web/src/main/webapp/index.jsp index 692590aed..52e81a84c 100755 --- a/karma-web/src/main/webapp/index.jsp +++ b/karma-web/src/main/webapp/index.jsp @@ -189,7 +189,13 @@ and related projects, please see: http://www.isi.edu/integration
  •  Name or ID First
    • - +
  • Semantic Labeling Service + +
    • +
  • Karma Client Name
    • @@ -409,6 +415,8 @@ and related projects, please see: http://www.isi.edu/integration var DEFAULT_PROPERTY_URI = "<%=ModelingConfigurationRegistry.getInstance().getModelingConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).getDefaultProperty()%>"; var showRDFSLabel_LabelFirst = <%=UIConfigurationRegistry.getInstance().getUIConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).showRDFSLabelWithLabelFirst()%>; var showRDFSLabel_IDFirst = <%=UIConfigurationRegistry.getInstance().getUIConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).showRDFSLabelWithIDFirst()%>; + var isSemanticLabelingOnline = <%=ModelingConfigurationRegistry.getInstance().getModelingConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).getOnlineSemanticTypingEnabled()%>; + var karmaClientName = "<%=ModelingConfigurationRegistry.getInstance().getModelingConfiguration(ContextParametersRegistry.getInstance().getDefault().getId()).getKarmaClientName()%>"; $(function() { @@ -506,7 +514,9 @@ and related projects, please see: http://www.isi.edu/integration showModeHeader("Automatic Mode"); loadPropertiesForCache(); + $("#modal_setKarmaClientName input[type=text]").val(karmaClientName); Settings.getInstance().setDisplayRDFSLabel(showRDFSLabel_LabelFirst, showRDFSLabel_IDFirst); + Settings.getInstance().setIsSemanticLabelingOnline(isSemanticLabelingOnline); }); var footerPositionTimer = null; diff --git a/karma-web/src/main/webapp/js/settings.js b/karma-web/src/main/webapp/js/settings.js index f50b28254..7ee16468e 100644 --- a/karma-web/src/main/webapp/js/settings.js +++ b/karma-web/src/main/webapp/js/settings.js @@ -6,7 +6,9 @@ var Settings = (function() { var showRDFSLabel = false; var showRDFSLabel_labelFirst = false; var showRDFSLabel_idFirst = false; - + + var isSemanticLabeling_online = false; + function init() { $("#displayRDFSLabel_labelFirst").on("click", function(e) { label_first = !$("#displayRDFSLabel_labelFirst span").is(":visible"); @@ -24,6 +26,46 @@ var Settings = (function() { label_first = showRDFSLabel_labelFirst; setDisplayRDFSLabel(label_first, id_first, true); }); + + $("#displaySemanticLabeling_Online").on("click", function(e) { + setIsSemanticLabelingOnline(!$("#displaySemanticLabeling_Online span").is(":visible")); + }); + $("#displaySemanticLabeling_Offline").on("click", function(e) { + setIsSemanticLabelingOnline($("#displaySemanticLabeling_Offline span").is(":visible")); + }); + + $('#modal_setKarmaClientName').on('shown.bs.modal', function () { + $("#modal_setKarmaClientName input[type=text]").select(); + }) + + $("#modal_setKarmaClientName button[type=submit]").click(setKarmaClientName); + } + + // this function handles visual and backend of toggling semantic typing + function setIsSemanticLabelingOnline(isOnline){ + if (isOnline == null) + return + isSemanticLabeling_online = isOnline; + if (isOnline){ + $("#displaySemanticLabeling_Online span").show(); + $("#displaySemanticLabeling_Offline span").hide(); + } else { + $("#displaySemanticLabeling_Online span").hide(); + $("#displaySemanticLabeling_Offline span").show(); + } + + var info = generateInfoObject("", "", "ToggleOnlineSemanticTypingCommand"); + showWaitingSignOnScreen(); + sendRequest(info); + } + + // This function calls SetKarmaClientName command to set the client name + function setKarmaClientName(){ + var name = $("#modal_setKarmaClientName input[type=text]").val(); + var info = generateInfoObject("", "", "SetKarmaClientNameCommand"); + info.value = name; + showWaitingSignOnScreen(); + sendRequest(info); } function setDisplayRDFSLabel(showLabelFirst, showIDFirst, update) { @@ -89,7 +131,8 @@ var Settings = (function() { showRDFSLabel: showRDFSLabel, showRDFSLabelWithIdFirst: showRDFSLabelWithIdFirst, showRDFSLabelWithLabelFirst: showRDFSLabelWithLabelFirst, - getDisplayLabel: getDisplayLabel + getDisplayLabel: getDisplayLabel, + setIsSemanticLabelingOnline: setIsSemanticLabelingOnline }; }; diff --git a/karma-web/src/main/webapp/model.jsp b/karma-web/src/main/webapp/model.jsp index 776ae74a4..561bfa6ed 100644 --- a/karma-web/src/main/webapp/model.jsp +++ b/karma-web/src/main/webapp/model.jsp @@ -146,5 +146,23 @@ a.icon-remove:hover { + + \ No newline at end of file