package ie.dcu.apps.ist.views; import ie.dcu.apps.ist.AppWindow; import ie.dcu.apps.ist.PainterRegistry; import ie.dcu.apps.ist.event.*; import ie.dcu.apps.ist.widgets.*; import ie.dcu.apps.ist.labelling.*; import ie.dcu.segment.*; import ie.dcu.segment.annotate.*; import ie.dcu.segment.options.SegmenterOptionDialog; import ie.dcu.segment.painters.SegmentationPainter; import ie.dcu.swt.*; import ie.dcu.swt.event.*; import java.io.BufferedReader; import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.net.*; import java.util.ArrayList; import java.util.Properties; import java.util.logging.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringEscapeUtils; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.jface.action.*; import org.eclipse.jface.operation.*; import org.eclipse.jface.resource.ImageDescriptor; import org.eclipse.swt.SWT; import org.eclipse.swt.events.*; import org.eclipse.swt.graphics.FontMetrics; import org.eclipse.swt.graphics.GC; import org.eclipse.swt.graphics.Point; import org.eclipse.swt.layout.*; import org.eclipse.swt.widgets.*; import org.json.JSONArray; import org.json.JSONObject; public class SegmentationView extends Composite { // Logger private static final Logger log = Logger.getLogger(SegmentationView.class.getName()); // Segmentation view properties private final Properties props; private final Properties curatornames; // Houses various painters private final PainterRegistry painters; // View handling annotations (image pane) private final AnnotatedImageControl view; // tool bars private final ToolBar mainToolbar, viewSelectionToolbar; // Composite for Term details private Composite termLookupComposite, termDetailComposite; // term detail table private static Table termDetailTable; // term synonym table (1-col list of nested synonyms) private static Table termSynonymTable; // Control to change brush size private final BrushControl brushControl; // Handles various events private final EventHandler eventHandler; // Proxy object to catch and log exceptions thrown by the segmentation // algorithm, without crashing the application. private RobustSegmenterProxy segmenterProxy; // Auto segment on update flag private boolean auto = true; // Inorder to toggle between segmentation and Labeling mode. Initially under Segmentation Mode private static boolean labelMode = false; // Combo box housing the selectable views private static Combo combo; public static Combo comboLabel,curatorCombo,speciesCombo; private static Button assign,searchSpecies; public static Text collectionId, comment; // Current segmentation tool private Segmenter segmenter; // Context to run the segmentation in, (may be null) private IRunnableContext runnableContext; // Flag to indicate that the runnable context blocks when fork is true private boolean blocksOnFork; // For getting the Labels through Web service private Labels labels; public enum Tool { SegmentMode(true), LabelMode(true), FormSegment(true), PredictSegments(false), ZoomIn(false), ZoomOut(false), ZoomOriginal(false), ZoomBestFit(false), Repaint(false), Undo(false), Redo(false), Clear(false), SetBrushSize(false), AutoApply(false), Apply(false), SetPainter(false), SetLabel(false), AssignButton(false), SetSpecies(false), SearchSpeciesButton(false), SetCurator(false), SegmenterOptions(false); private ToolAction action; private boolean isTextLabel; Tool(boolean isTextLabel) { this.isTextLabel = isTextLabel; } }; public enum termDetailLabels { name("Name"), accession_id("Accession ID"), aspect("Branch") /* a.k.a. "Aspect" */, definition("Definition"), comment("Comment"); final String extendedLabel; termDetailLabels(String extendedLabel) { this.extendedLabel = extendedLabel; } }; // constructor public SegmentationView(Properties props,Properties curatornames,Composite parent, int style) { super(parent, style); this.props = props; this.curatornames = curatornames; painters = new PainterRegistry(); mainToolbar = new ToolBar(this, SWT.RIGHT | SWT.FLAT); viewSelectionToolbar = new ToolBar(this, SWT.RIGHT | SWT.FLAT); termLookupComposite = new Composite(this, SWT.RIGHT | SWT.FLAT); view = new AnnotatedImageControl(this, SWT.BORDER); termDetailComposite = new Composite(this, SWT.RIGHT | SWT.FLAT); brushControl = new BrushControl(getShell(), SWT.BORDER); eventHandler = new EventHandler(); segmenterProxy = new RobustSegmenterProxy(); labels = new Labels(); init(); } /** * Initialize. */ private void init() { initTools(); //create all visual controls createToolMainToolbar(); createToolViewSelectionToolbar(); createToolTermLookupComposite(); createTermDetailTable(); createTermSynonymTable(); createTermDetailFields(); // lay out the controls layoutControls(); updatePainters(); addListeners(); updateToolStates(); } private void initTools() { for (Tool t : Tool.values()) { if (t.action == null) { new ToolAction(t); } } } private void addListeners() { brushControl.addSelectionListener(eventHandler); if(!getLabelMode()) { view.addContextChangeListener(eventHandler); } view.addZoomListener(eventHandler); addDisposeListener(eventHandler); } private void createToolMainToolbar() { SwtUtils.addButton(mainToolbar, 300, getAction(Tool.SetPainter).getText()); ToolBarManager m = new ToolBarManager(mainToolbar); m.add(getAction(Tool.SegmentMode)); m.add(getAction(Tool.LabelMode)); m.add(getAction(Tool.FormSegment)); m.add(new Separator()); m.add(getAction(Tool.ZoomIn)); m.add(getAction(Tool.ZoomOut)); m.add(getAction(Tool.ZoomOriginal)); m.add(getAction(Tool.ZoomBestFit)); m.add(new Separator()); m.add(getAction(Tool.Repaint)); m.add(getAction(Tool.Undo)); m.add(getAction(Tool.Redo)); m.add(getAction(Tool.Clear)); m.add(new Separator()); m.add(getAction(Tool.SetBrushSize)); m.add(new Separator()); m.update(true); } private void createToolViewSelectionToolbar() { SwtUtils.addLabel(viewSelectionToolbar, getAction(Tool.SetPainter).getText()); combo = SwtUtils.addCombo(viewSelectionToolbar, 115, SWT.READ_ONLY); combo.setToolTipText( getAction(Tool.SetPainter).getToolTipText()); combo.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent e) { execute(Tool.SetPainter, null); } }); viewSelectionToolbar.pack(); } private void resetViewSelectionToolbar(SegmentationPainter painter) { this.setPainter(painter); combo.setText(painter.getName()); } /* * used to clear the view for closing a file, switching files, or opening a new file; * removes all in-memory instances of segmentation data and metadata; * TODO: then recall this method from the Open (Open Recent), Go Next, Go Previous, and Close...when you add Close */ public void resetView() { Tool.LabelMode.action.setChecked(false); resetViewSelectionToolbar(painters.get("Segmented")); // clear metadata comboLabel.removeAll(); comboLabel.setEnabled(false); clearTermDetailTable(); clearTermSynonymTable(); //collectionID.setLayoutData(); collectionId.setText(""); comment.setText(""); curatorCombo.deselectAll(); speciesCombo.removeAll(); // clear segment data (term labels, layers, image coords) if (view.getContext() != null) { this.getContext().nullifySegments(); this.getContext().nullifySegments(); // TODO: why twice? this.setContext(null); SegmentationMask.numberOfLayers = 0; // reset the layer counter for the next image } } private String parseAccessionIdFromComboLabel(Combo comboLabel) { return comboLabel.getText().substring(comboLabel.getText().indexOf('{')+1,comboLabel.getText().length()-1); } /** * Third tool bar for holding the Annotate Combo box. */ private void createToolTermLookupComposite() { /*SwtUtils.addLabelToComposite(termLookupComposite, getAction(Tool.SetOntologyLabel).getText()); comboLabel = SwtUtils.addComboToComposite(termLookupComposite, 250, SWT.SIMPLE); comboLabel.setToolTipText( getAction(Tool.SetOntologyLabel).getToolTipText()); comboLabel.addKeyListener(new KeyAdapter() { public void keyReleased(KeyEvent e) { dropdownLabelsFromJSON(e,comboLabel); } });*/ SwtUtils.addLabelToComposite(termLookupComposite, getAction(Tool.SetLabel).getText()); comboLabel = SwtUtils.addComboToComposite(termLookupComposite, 250, SWT.SIMPLE); comboLabel.setToolTipText( getAction(Tool.SetLabel).getToolTipText()); comboLabel.addKeyListener(new KeyAdapter() { public void keyReleased(KeyEvent e) { dropdownLabelsFromJSON(e,comboLabel); } }); comboLabel.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent e) { termDetailLookup(parseAccessionIdFromComboLabel(comboLabel)); } }); assign = SwtUtils.addButtonToComposite(termLookupComposite, 75, "Assign"); assign.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent arg0) { if(view.getContext().isEnabled()) { OntologyTerm term = new OntologyTerm(); term.setName((String)comboLabel.getData(formatizeToUnescapeHTML(comboLabel.getText()))); term.setAccessionId(parseAccessionIdFromComboLabel(comboLabel)); view.getContext().getEnabledMask().ontologyTerm = term; } execute(Tool.AssignButton, null); } }); } public static String formatizeToUnescapeHTML(String term) { Pattern pattern = Pattern.compile("&#[0-9]+"); Matcher matcher = pattern.matcher(term); StringBuffer formattedTerm = new StringBuffer(); while (matcher.find()) { String matchedText = matcher.group(); matcher.appendReplacement(formattedTerm, matchedText + ";"); } matcher.appendTail(formattedTerm); return formattedTerm.toString(); } /** * For dropping down the labels on pressing the down arrow key just like "Google suggests" functionality. * @param e */ public void dropdownLabelsFromJSON(KeyEvent e, Combo combo) { ArrayList terms = new ArrayList(); //For the down arrow functionality if(e.keyCode == 16777218) { combo.setListVisible(true); } // If key pressed is only a number of character or space. else if ((e.keyCode >= 48 && e.keyCode <= 57) || (e.keyCode >= 97 && e.keyCode <= 122) || e.keyCode == 32) { //For removing all previously assigned labels combo.remove(0,combo.getItemCount()-1); terms = labels.getOntologyTerms(combo.getText()); assign.setEnabled(!(combo.getText().isEmpty())); } int i = 0; for (OntologyTerm term : terms) { // set text for term label combo.add(term.getFormattedTerm(),i); combo.setData(term.getFormattedTerm(), term.getName()); i++; } } /** * For dropping down the labels on pressing the down arrow key just like "Google suggests" functionality. * @param e */ public void dropdownLabelsFromXML(SelectionEvent e) { ArrayList speciesTerms = new ArrayList(); speciesTerms = labels.getSpeciesTerms(speciesCombo.getText()); String text = speciesCombo.getText(); speciesCombo.removeAll(); speciesCombo.setText(text); int i = 0; for (SpeciesTerm speciesTerm : speciesTerms) { speciesCombo.add(speciesTerm.getSpeciesName(),i); speciesCombo.setData(speciesTerm.getSpeciesName(), speciesTerm.getSpeciesId()); i++; } searchSpecies.setEnabled(!(speciesCombo.getText().isEmpty())); speciesCombo.setListVisible(true); } /* * call term detail web service method using accession id from 1) segment obj or 2) label selection * (before assign button is clicked) */ public static void termDetailLookup(String accessionId) { termDetailTable.setEnabled(true); termSynonymTable.setEnabled(true); termSynonymTable.removeAll(); String webServiceURL = new String(); try { String encodedContent = URLEncoder.encode(accessionId.toString(),"UTF-8"); webServiceURL = AppWindow.props.getProperty("POWebService.TermDetail.URL"); webServiceURL = webServiceURL.replace("", encodedContent); URL url = new URL(webServiceURL); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); // if the response from Web service is 'OK' if (connection.getResponseCode() == 200) { String inputLine; JSONObject responseObject = null; while ((inputLine = in.readLine()) != null) { responseObject = new JSONObject(inputLine); } JSONArray responseArray = new JSONArray(responseObject.getString("PO_term_detail_response")); // set the table values termDetailLabels[] values = termDetailLabels.values(); TableItem[] items = termDetailTable.getItems(); for(int i=0; i 0; } return false; } public Action getAction(Tool t) { return t.action; } public void showSegmenterOptions() { if (canShowOptions()) { SegmenterOptionDialog dialog = new SegmenterOptionDialog(getShell(), segmenter); int result = dialog.open(); switch (result) { case SegmenterOptionDialog.OK: // Re-segment apply(); } } } public boolean hasSegmenter() { return segmenter != null; } public void performSegmentation(AnnotationEvent e, SegmentationContext ctx) { if (runnableContext == null) { segment(e, ctx); } else { segmentWithRunnableContext(e, ctx); } // Repaint everything repaint(); } public void setRunnableContext(IRunnableContext context, boolean blocksOnFork) { runnableContext = context; this.blocksOnFork = blocksOnFork; } public void addContextChangeListener(ContextChangeListener listener) { view.addContextChangeListener(listener); } public void removeContextChangeListener(ContextChangeListener listener) { view.removeContextChangeListener(listener); } public void setEnabled(boolean enabled) { super.setEnabled(enabled); mainToolbar.setEnabled(enabled); viewSelectionToolbar.setEnabled(enabled); view.setEnabled(enabled); updateToolStates(); } private void run(boolean fork, IRunnableWithProgress runnable) { boolean wasEnabled = isEnabled(); try { // Disable interactions setEnabled(false); // Run segmentation runnableContext.run(fork, false, runnable); } catch (InvocationTargetException ex) { // Can't happen without a runtime exception occurring, so re-wrap throw new RuntimeException(ex); } catch (InterruptedException ex) { // Can't happen because cancellable is false throw new RuntimeException(ex); } finally { setEnabled(wasEnabled); } } private void segmentWithRunnableContext( final AnnotationEvent e, final SegmentationContext ctx) { // Fork if the segmenter is slow, and only if our runnable context blocks boolean fork = blocksOnFork /*&& !segmenter.isFast()*/; run(fork, new IRunnableWithProgress() { public void run(IProgressMonitor monitor) { monitor.beginTask("Segmenting", IProgressMonitor.UNKNOWN); try { segment(e, ctx); } finally { monitor.done(); } } }); } private void start(final SegmentationContext ctx) { if (ctx != null && hasSegmenter()) { if (runnableContext == null) { // No runnable context segmenterProxy.init(ctx); segmenterProxy.update(ctx); } else { // Run with progress monitor run(blocksOnFork, new IRunnableWithProgress() { public void run(IProgressMonitor monitor) { monitor.beginTask("Initializing", IProgressMonitor.UNKNOWN); try { segmenterProxy.init(ctx); segmenterProxy.update(ctx); } finally { monitor.done(); } } }); } // Repaint everything repaint(); } } private void segment(AnnotationEvent e, SegmentationContext ctx) { if (hasSegmenter()) { if (e == null) { // No event triggered this, so do a full update segmenterProxy.update(ctx); } else { // Call appropriate segmenter function switch (e.type) { case Cleared: segmenterProxy.update(ctx); break; case Undone: segmenterProxy.removed(ctx, e.annotation); break; case Redone: case Added: segmenterProxy.added(ctx, e.annotation); break; default: throw new RuntimeException(); } } } } private String getText(Tool a) { return props.getProperty(getKey(a, "text")); } private String getToolTip(Tool a) { return props.getProperty(getKey(a, "tooltip")); } private ImageDescriptor getImage(Tool a) { String location = props.getProperty(getKey(a, "image")); if (location != null) { if (location.length() == 0) { return null; } try { return ImageDescriptor.createFromURL(new URL(location)); } catch (MalformedURLException e) { return null; } } return null; } private static String getKey(Tool action, String object) { return String.format("SegmentationView.Action.%s.%s", action, object); } private int getType(Tool a) { switch (a) { case SegmentMode: case LabelMode: case AutoApply: return Action.AS_CHECK_BOX; case SetBrushSize: return Action.AS_DROP_DOWN_MENU; case SetPainter: return Action.AS_UNSPECIFIED; default: return Action.AS_PUSH_BUTTON; } } private void execute(Tool a, Event e) { switch (a) { case SegmentMode: setAnnotationType(AnnotationType.Foreground); setLabellingMode(false); break; case LabelMode: setLabellingMode(true); break; case FormSegment: formSegmetationObject(); break; case ZoomIn: zoomIn(); break; case ZoomOut: zoomOut(); break; case ZoomOriginal: zoomOriginal(); break; case ZoomBestFit: zoomBestFit(); break; case Repaint: repaint(); break; case Undo: undo(); break; case Redo: redo(); break; case Clear: clear(); break; case SetBrushSize: showBrushControl(e); break; case AutoApply: setAutoApply(a.action.isChecked()); break; case Apply: apply(); break; case SetPainter: setPainter(); break; case AssignButton: assignLabel(); break; } updateToolStates(); } private void updateToolStates() { if (isEnabled()) { Tool.Apply.action.setEnabled(!Tool.AutoApply.action.isChecked()); Tool.Undo.action.setEnabled(canUndo()); Tool.Redo.action.setEnabled(canRedo()); Tool.ZoomIn.action.setEnabled(canZoomIn()); Tool.ZoomOut.action.setEnabled(canZoomOut()); Tool.ZoomOriginal.action.setEnabled(canZoomOriginal()); Tool.ZoomBestFit.action.setEnabled(canZoomBestFit()); if(view.getContext() != null) { // SegmentMasks are formed and is in labelmode and no halfdone annotations are conditions for enabling the comboLabel comboLabel.setEnabled((view.getContext().getSegmentationMasks().size() > 0) && getLabelMode() && !(view.getContext().getAnnotations().canUndo())); //assign.setEnabled(false); } else { comboLabel.setEnabled(false); } assign.setEnabled(canZoomBestFit() & !(comboLabel.getText().isEmpty()) && comboLabel.isEnabled()); termDetailTable.setEnabled(assign.isEnabled()); termSynonymTable.setEnabled(assign.isEnabled()); Tool.SegmentMode.action.setChecked(!getLabelMode()); Tool.LabelMode.action.setChecked(getLabelMode()); Tool.Clear.action.setEnabled(canClear()); Tool.AutoApply.action.setChecked(auto); Tool.SegmentMode.action.setEnabled(canZoomIn()); Tool.LabelMode.action.setEnabled(canZoomIn()); Tool.FormSegment.action.setEnabled(CanLabel() && !(Tool.LabelMode.action.isChecked())); // Always enabled if view enabled Tool.SetBrushSize.action.setEnabled(true); Tool.SetPainter.action.setEnabled(true); Tool.Repaint.action.setEnabled(true); Tool.AutoApply.action.setEnabled(true); } else { // Everything disabled for (Tool t : Tool.values()) { t.action.setEnabled(false); } } } private void setPainter() { SegmentationPainter painter = painters.get(combo.getText()); setPainter(painter); } /* Assign the label to the segment on clicking assign button and update the image segment. */ private void assignLabel() { SegmentationPainter painter = painters.get("Segmented"); setPainter(painter); } private void apply() { performSegmentation(null, view.getContext()); } private void showBrushControl(Event e) { brushControl.showBelow(mainToolbar, (ToolItem) e.widget); } /** * Handles a context change in the view. * * @param e * The event. */ private void handleContextChanged(ContextChangedEvent e) { if (e.oldContext != null) { e.oldContext.removeAnnotationListener(eventHandler); // Finish old segmentation segmenterProxy.finish(e.oldContext); } if (e.newContext != null) { e.newContext.addAnnotationListener(eventHandler); // Begin new segmentation (can cause deferred events // to be processed, so we async exec to ensure that // the other context change handlers are called before // deferred events are processed) final SegmentationContext ctx = e.newContext; getDisplay().asyncExec(new Runnable() { public void run() { start(ctx); } }); } updateToolStates(); } /** * Cleans up resources when the view is disposed. */ private void handleDisposed() { SegmentationContext ctx = view.getContext(); if (ctx != null && !ctx.isDisposed()) { // Finish any segmentation segmenterProxy.finish(ctx); // Dispose context ctx.dispose(); } // Dispose all painters painters.dispose(); } public static boolean getLabelMode() { return labelMode; } public static void setLabelMode(boolean labelMode) { SegmentationView.labelMode = labelMode; } /** * Tool bar action. Delegates running to the execute method. */ private class ToolAction extends Action { private final Tool tool; public ToolAction(Tool tool) { super(SegmentationView.this.getText(tool), getType(tool)); this.tool = tool; setToolTipText(SegmentationView.this.getToolTip(tool)); if(tool.isTextLabel) { setText(SegmentationView.this.getText(tool)); } else { setImageDescriptor(SegmentationView.this.getImage(tool)); } tool.action = this; } public void runWithEvent(Event e) { execute(tool, e); } } /** * Handles various events coming from the view and toolbar controls. * */ private final class EventHandler extends SelectionAdapter implements ZoomListener, SelectionListener, ContextChangeListener, AnnotationListener, DisposeListener { public void widgetDisposed(DisposeEvent e) { handleDisposed(); } public void zoomChanged(ZoomEvent e) { updateToolStates(); } public void widgetSelected(SelectionEvent e) { setBrushSize(brushControl.getBrushSize()); } public void contextChanged(ContextChangedEvent e) { handleContextChanged(e); } public void annotationPerformed(AnnotationEvent e) { if (!isEnabled()) { log.warning("annotation performed while not enabled"); } if (auto && isEnabled()) { performSegmentation(e, view.getContext()); } updateToolStates(); } public void annotationRedone(AnnotationEvent e) { if (!isEnabled()) { log.warning("annotation redone while not enabled"); } if (auto && isEnabled()) { performSegmentation(e, view.getContext()); } updateToolStates(); } public void annotationUndone(AnnotationEvent e) { if (!isEnabled()) { log.warning("annotation undone while not enabled"); } if (auto && isEnabled()) { performSegmentation(e, view.getContext()); } updateToolStates(); } public void annotationsCleared(AnnotationEvent e) { if (!isEnabled()) { log.warning("annotations cleared while not enabled"); } if (auto && isEnabled()) { performSegmentation(e, view.getContext()); } updateToolStates(); } }; /** * Class that prevents segmentation algorithms crashing the application * by catching any thrown exceptions and logging them. * * @author Kevin McGuinness */ private class RobustSegmenterProxy { public void init(SegmentationContext ctx) { if (segmenter != null) { try { segmenter.init(ctx); } catch (Throwable th) { severe(th, "%s.init()", getSegmenterClassName()); } } } public void update(SegmentationContext ctx) { if (segmenter != null) { try { segmenter.update(ctx); } catch (Throwable th) { severe(th, "%s.update()", getSegmenterClassName()); } } } public void added(SegmentationContext ctx, Annotation a) { if (segmenter != null) { try { segmenter.added(ctx, a); } catch (Throwable th) { severe(th, "%s.added()", getSegmenterClassName()); } } } public void removed(SegmentationContext ctx, Annotation a) { if (segmenter != null) { try { segmenter.removed(ctx, a); } catch (Throwable th) { severe(th, "%s.removed()", getSegmenterClassName()); } } } public void finish(SegmentationContext ctx) { if (segmenter != null) { try { segmenter.finish(ctx); } catch (Throwable th) { severe(th, "%s.finish()", getSegmenterClassName()); } } } public String getSegmenterClassName() { return segmenter.getClass().getSimpleName(); } private void severe(Throwable th, String message, Object ... args) { log.log(Level.SEVERE, String.format(message, args), th); } } }