From 37b2c9538797aea6cc55678f2f031f4be6981651 Mon Sep 17 00:00:00 2001 From: nsi Date: Wed, 13 Apr 2016 14:01:55 +0100 Subject: [PATCH 01/14] Add serialization --- .../com/fulmicoton/multiregexp/MultiPatternAutomaton.java | 5 ++++- .../com/fulmicoton/multiregexp/MultiPatternSearcher.java | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java index 43565b0..c35aea5 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java @@ -1,5 +1,6 @@ package com.fulmicoton.multiregexp; +import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; @@ -11,7 +12,9 @@ import dk.brics.automaton.DkBricsAutomatonHelper; import dk.brics.automaton.State; -public class MultiPatternAutomaton { +public class MultiPatternAutomaton implements Serializable { + + private static final long serialVersionUID = -8269666436361824366L; public final int[][] accept; final boolean[] atLeastOneAccept; diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java index ec98398..a125b41 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java @@ -6,12 +6,15 @@ import dk.brics.automaton.StatePair; import dk.brics.automaton.Transition; +import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -public class MultiPatternSearcher { +public class MultiPatternSearcher implements Serializable { + + private static final long serialVersionUID = -1812442985139693661L; private final MultiPatternAutomaton automaton; private final List individualAutomatons; From e23c406e27c70dc741b979a9eeb60e577fe566dd Mon Sep 17 00:00:00 2001 From: nsi Date: Wed, 13 Apr 2016 18:55:12 +0100 Subject: [PATCH 02/14] Add ability to use non-tableized automaton. This is slower (I think around half the speed for search) but requires much less memory. I have observed memory usage of 1% the tableized version. --- .../com/fulmicoton/multiregexp/MultiPattern.java | 9 +++++++-- .../multiregexp/MultiPatternSearcher.java | 13 ++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java index a65a0b3..04853ff 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java @@ -34,6 +34,11 @@ public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix) { return MultiPatternAutomaton.make(automata); } + public MultiPatternSearcher searcher() + { + return searcher(true); + } + /** * Equivalent of Pattern.compile, but the result is only valid for pattern search. * The searcher will return the first occurrence of a pattern. @@ -43,7 +48,7 @@ public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix) { * * @return A searcher object */ - public MultiPatternSearcher searcher() { + public MultiPatternSearcher searcher(boolean tableize) { final MultiPatternAutomaton searcherAutomaton = makeAutomatonWithPrefix(".*"); final List indidivualAutomatons = new ArrayList<>(); for (final String pattern: this.patterns) { @@ -52,7 +57,7 @@ public MultiPatternSearcher searcher() { automaton.determinize(); indidivualAutomatons.add(automaton); } - return new MultiPatternSearcher(searcherAutomaton, indidivualAutomatons); + return new MultiPatternSearcher(searcherAutomaton, indidivualAutomatons, tableize); } diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java index a125b41..424b921 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java @@ -21,16 +21,23 @@ public class MultiPatternSearcher implements Serializable { private final List inverseAutomatons; MultiPatternSearcher(final MultiPatternAutomaton automaton, - final List individualAutomatons) { + final List individualAutomatons) + { + this(automaton, individualAutomatons, true); + } + + MultiPatternSearcher(final MultiPatternAutomaton automaton, + final List individualAutomatons, + boolean tableize) { this.automaton = automaton; this.individualAutomatons = new ArrayList<>(); for (final Automaton individualAutomaton: individualAutomatons) { - this.individualAutomatons.add(new RunAutomaton(individualAutomaton)); + this.individualAutomatons.add(new RunAutomaton(individualAutomaton, tableize)); } this.inverseAutomatons = new ArrayList<>(this.individualAutomatons.size()); for (final Automaton individualAutomaton: individualAutomatons) { final Automaton inverseAutomaton = inverseAutomaton(individualAutomaton); - this.inverseAutomatons.add(new RunAutomaton(inverseAutomaton)); + this.inverseAutomatons.add(new RunAutomaton(inverseAutomaton, tableize)); } } From 0997a30b51910e1d6f76cc772e3bd98606f0caa2 Mon Sep 17 00:00:00 2001 From: nsi Date: Thu, 14 Apr 2016 00:01:52 +0100 Subject: [PATCH 03/14] Spelling correction --- src/main/java/com/fulmicoton/multiregexp/MultiPattern.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java index 04853ff..a981f09 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java @@ -50,14 +50,14 @@ public MultiPatternSearcher searcher() */ public MultiPatternSearcher searcher(boolean tableize) { final MultiPatternAutomaton searcherAutomaton = makeAutomatonWithPrefix(".*"); - final List indidivualAutomatons = new ArrayList<>(); + final List individualAutomatons = new ArrayList<>(); for (final String pattern: this.patterns) { final Automaton automaton = new RegExp(pattern).toAutomaton(); automaton.minimize(); automaton.determinize(); - indidivualAutomatons.add(automaton); + individualAutomatons.add(automaton); } - return new MultiPatternSearcher(searcherAutomaton, indidivualAutomatons, tableize); + return new MultiPatternSearcher(searcherAutomaton, individualAutomatons, tableize); } From 93cb31ae825fa329a42b809a77b9dbcce0d3db29 Mon Sep 17 00:00:00 2001 From: nsi Date: Thu, 14 Apr 2016 19:28:06 +0100 Subject: [PATCH 04/14] Add default method for multithreaded make in MultiPatternAutomaton. This creates the same number of threads as there are processors available. For small numbers of patterns this makes little difference, however testing on my Mac Pro with 8 processors, with large numbers of patterns (1,000 - 5,000) the multithreaded make uses 4-6 threads and is around 3 to 4 times faster. --- .../fulmicoton/multiregexp/MultiPattern.java | 15 ++- .../multiregexp/MultiPatternAutomaton.java | 121 +++++++++++++++++- 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java index a981f09..86e372b 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java @@ -31,12 +31,7 @@ public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix) { automaton.minimize(); automata.add(automaton); } - return MultiPatternAutomaton.make(automata); - } - - public MultiPatternSearcher searcher() - { - return searcher(true); + return MultiPatternAutomaton.multithreadedMake(automata); } /** @@ -48,7 +43,13 @@ public MultiPatternSearcher searcher() * * @return A searcher object */ - public MultiPatternSearcher searcher(boolean tableize) { + public MultiPatternSearcher searcher() + { + return searcher(true); + } + + + public MultiPatternSearcher searcher(final boolean tableize) { final MultiPatternAutomaton searcherAutomaton = makeAutomatonWithPrefix(".*"); final List individualAutomatons = new ArrayList<>(); for (final String pattern: this.patterns) { diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java index c35aea5..7b17957 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java @@ -2,17 +2,22 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; import dk.brics.automaton.Automaton; import dk.brics.automaton.DkBricsAutomatonHelper; import dk.brics.automaton.State; -public class MultiPatternAutomaton implements Serializable { +public class MultiPatternAutomaton + implements Serializable { private static final long serialVersionUID = -8269666436361824366L; @@ -59,6 +64,120 @@ static MultiState initialState(List automata) { return new MultiState(initialStates); } + static MultiPatternAutomaton multithreadedMake(final List automata) { + for (final Automaton automaton: automata) { + automaton.determinize(); + } + + final char[] points = DkBricsAutomatonHelper.pointsUnion(automata); + + // states that are still to be visited + final Queue statesToVisits = new ConcurrentLinkedQueue<>(); + final MultiState initialState = initialState(automata); + statesToVisits.add(initialState); + + final Map transitionMap = new ConcurrentHashMap<>(); + + final Map multiStateIndex = new ConcurrentHashMap<>(); + multiStateIndex.put(initialState, 0); + + final int numberOfThreads = Runtime.getRuntime().availableProcessors(); + + final Object lockObject = new Object(); + final List activeThreads = Collections.synchronizedList(new ArrayList()); + final CountDownLatch doneSignal = new CountDownLatch(numberOfThreads); + for (int thread = 0; thread < numberOfThreads; thread++) { + new Thread(new Runnable() { + @Override + public void run() { + activeThreads.add(Thread.currentThread()); + + while (true) { + MultiState visitingState; + while ((visitingState = statesToVisits.poll()) != null) { + assert multiStateIndex.containsKey(visitingState); + + int stateId = multiStateIndex.get(visitingState); + final int[] curTransitions = new int[points.length]; + transitionMap.put(stateId, curTransitions); + + for (int c = 0; c < points.length; c++) { + final char point = points[c]; + final MultiState destState = visitingState.step(point); + if (destState.isNull()) { + curTransitions[c] = -1; + } else { + final int destStateId; + synchronized (multiStateIndex) { + if (!multiStateIndex.containsKey(destState)) { + destStateId = multiStateIndex.size(); + multiStateIndex.put(destState, destStateId); + statesToVisits.add(destState); + synchronized (lockObject) { + // wake threads to process destState + lockObject.notifyAll(); + } + } else { + destStateId = multiStateIndex.get(destState); + } + } + curTransitions[c] = destStateId; + } + } + } + activeThreads.remove(Thread.currentThread()); + // if there are no active threads then we are done + if (activeThreads.isEmpty()) { + synchronized (lockObject) { + // wake waiting threads so they can end + lockObject.notifyAll(); + } + // end this thread + break; + } else { + synchronized (lockObject) { + try { + lockObject.wait(); + } + catch (InterruptedException ignore) { + } + } + // if there are no active threads then we are done + if (activeThreads.isEmpty()) { + // end this thread + break; + } else { + activeThreads.add(Thread.currentThread()); + } + } + } + doneSignal.countDown(); + } + }).start(); + } + try { + // wait for all to finish + doneSignal.await(); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + + assert transitionMap.size() == multiStateIndex.size(); + + final int[] transitions = new int[transitionMap.size() * points.length]; + for (final Map.Entry entry : transitionMap.entrySet()) { + System.arraycopy(entry.getValue(), 0, transitions, entry.getKey() * points.length, points.length); + } + + final int[][] acceptValues = new int[multiStateIndex.size()][]; + for (final Map.Entry entry : multiStateIndex.entrySet()) { + acceptValues[entry.getValue()] = entry.getKey().toAcceptValues(); + } + + return new MultiPatternAutomaton(acceptValues, transitions, points, automata.size()); + } + static MultiPatternAutomaton make(final List automata) { for (final Automaton automaton: automata) { automaton.determinize(); From d627b74483d8772254a80138abcaa5feb462f591 Mon Sep 17 00:00:00 2001 From: nsi Date: Fri, 15 Apr 2016 19:02:08 +0100 Subject: [PATCH 05/14] Optimisations - remove assert from while loop call notify rather than notifyAll as we only need to wake a single thread don't check for whether process has finished on each thread wakeup --- .../multiregexp/MultiPatternAutomaton.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java index 7b17957..9ec0756 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java @@ -95,7 +95,7 @@ public void run() { while (true) { MultiState visitingState; while ((visitingState = statesToVisits.poll()) != null) { - assert multiStateIndex.containsKey(visitingState); +// assert multiStateIndex.containsKey(visitingState); int stateId = multiStateIndex.get(visitingState); final int[] curTransitions = new int[points.length]; @@ -114,8 +114,8 @@ public void run() { multiStateIndex.put(destState, destStateId); statesToVisits.add(destState); synchronized (lockObject) { - // wake threads to process destState - lockObject.notifyAll(); + // wake a thread to process destState + lockObject.notify(); } } else { destStateId = multiStateIndex.get(destState); @@ -142,13 +142,7 @@ public void run() { catch (InterruptedException ignore) { } } - // if there are no active threads then we are done - if (activeThreads.isEmpty()) { - // end this thread - break; - } else { - activeThreads.add(Thread.currentThread()); - } + activeThreads.add(Thread.currentThread()); } } doneSignal.countDown(); From 26c90e49bb28d07689ccae545f783abcbe330629 Mon Sep 17 00:00:00 2001 From: nsi Date: Fri, 15 Apr 2016 19:36:28 +0100 Subject: [PATCH 06/14] Optimisations - replace hashmap contains and get with a single get. Note that the contains call will more often than not return false resulting in the need for a get --- .../com/fulmicoton/multiregexp/MultiPatternAutomaton.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java index 9ec0756..e248f3c 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java @@ -107,9 +107,10 @@ public void run() { if (destState.isNull()) { curTransitions[c] = -1; } else { - final int destStateId; + Integer destStateId; synchronized (multiStateIndex) { - if (!multiStateIndex.containsKey(destState)) { + destStateId = multiStateIndex.get(destState); + if (destStateId == null) { destStateId = multiStateIndex.size(); multiStateIndex.put(destState, destStateId); statesToVisits.add(destState); @@ -117,8 +118,6 @@ public void run() { // wake a thread to process destState lockObject.notify(); } - } else { - destStateId = multiStateIndex.get(destState); } } curTransitions[c] = destStateId; From 5ab713e359a996c9c1876bd87fc446e7ed81ee2e Mon Sep 17 00:00:00 2001 From: nsi Date: Fri, 15 Apr 2016 19:38:49 +0100 Subject: [PATCH 07/14] Optimisations - remove pointless assignment --- .../java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java index e248f3c..970ff55 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java @@ -102,8 +102,7 @@ public void run() { transitionMap.put(stateId, curTransitions); for (int c = 0; c < points.length; c++) { - final char point = points[c]; - final MultiState destState = visitingState.step(point); + final MultiState destState = visitingState.step(points[c]); if (destState.isNull()) { curTransitions[c] = -1; } else { From 2a9b3f610f3dd1834e737059e1f6c811bd06b3f7 Mon Sep 17 00:00:00 2001 From: nsi Date: Fri, 15 Apr 2016 20:02:46 +0100 Subject: [PATCH 08/14] trivial - add final to variable --- .../java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java index 970ff55..eaaadd5 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternAutomaton.java @@ -97,8 +97,8 @@ public void run() { while ((visitingState = statesToVisits.poll()) != null) { // assert multiStateIndex.containsKey(visitingState); - int stateId = multiStateIndex.get(visitingState); final int[] curTransitions = new int[points.length]; + final int stateId = multiStateIndex.get(visitingState); transitionMap.put(stateId, curTransitions); for (int c = 0; c < points.length; c++) { From 0b2a81314df14c6a3fa235d8717ef9b16a147907 Mon Sep 17 00:00:00 2001 From: nsi Date: Sat, 16 Apr 2016 03:37:52 +0100 Subject: [PATCH 09/14] Only calculate pattern start and end if those methods are called, thus optimising the "next()"" method. Also provide methods to return all the matching patterns (and their starts and end), still default is to return the first pattern. --- .../multiregexp/MultiPatternSearcher.java | 167 +++++++++++------- 1 file changed, 103 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java index 424b921..11f88b6 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java @@ -12,7 +12,8 @@ import java.util.List; import java.util.Map; -public class MultiPatternSearcher implements Serializable { +public class MultiPatternSearcher + implements Serializable { private static final long serialVersionUID = -1812442985139693661L; @@ -21,8 +22,7 @@ public class MultiPatternSearcher implements Serializable { private final List inverseAutomatons; MultiPatternSearcher(final MultiPatternAutomaton automaton, - final List individualAutomatons) - { + final List individualAutomatons) { this(automaton, individualAutomatons, true); } @@ -31,11 +31,11 @@ public class MultiPatternSearcher implements Serializable { boolean tableize) { this.automaton = automaton; this.individualAutomatons = new ArrayList<>(); - for (final Automaton individualAutomaton: individualAutomatons) { + for (final Automaton individualAutomaton : individualAutomatons) { this.individualAutomatons.add(new RunAutomaton(individualAutomaton, tableize)); } this.inverseAutomatons = new ArrayList<>(this.individualAutomatons.size()); - for (final Automaton individualAutomaton: individualAutomatons) { + for (final Automaton individualAutomaton : individualAutomatons) { final Automaton inverseAutomaton = inverseAutomaton(individualAutomaton); this.inverseAutomatons.add(new RunAutomaton(inverseAutomaton, tableize)); } @@ -43,11 +43,11 @@ public class MultiPatternSearcher implements Serializable { static Automaton inverseAutomaton(final Automaton automaton) { final Map stateMapping = new HashMap<>(); - for (final State state: automaton.getStates()) { + for (final State state : automaton.getStates()) { stateMapping.put(state, new State()); } - for (final State state: automaton.getStates()) { - for (final Transition transition: state.getTransitions()) { + for (final State state : automaton.getStates()) { + for (final Transition transition : state.getTransitions()) { final State invDest = stateMapping.get(state); final State invOrig = stateMapping.get(transition.getDest()); invOrig.addTransition(new Transition(transition.getMin(), transition.getMax(), invDest)); @@ -58,7 +58,7 @@ static Automaton inverseAutomaton(final Automaton automaton) { final State initialState = new State(); inverseAutomaton.setInitialState(initialState); final List epsilons = new ArrayList<>(); - for (final State acceptState: automaton.getAcceptStates()) { + for (final State acceptState : automaton.getAcceptStates()) { final State invOrigState = stateMapping.get(acceptState); final StatePair statePair = new StatePair(initialState, invOrigState); epsilons.add(statePair); @@ -77,30 +77,98 @@ public Cursor search(CharSequence s, int position) { public class Cursor { private final CharSequence seq; - private int matchingPattern = -1; - private int end = 0; - private int start = -1; + private int[] matchingPatterns = null; + private int[] matchingPatternsStart = null; + private int[] matchingPatternsEnd = null; + private int currentPosition = 0; Cursor(CharSequence seq, int position) { this.seq = seq; - this.end = position; + this.currentPosition = position; } public int start() { - return this.start; + return start(0); + } + + public int start(int patternIndex) { + if (this.matchingPatterns == null) { + return -1; + } + + if (this.matchingPatternsStart[patternIndex] == -1) { + // we rewind using the backward automaton to find the start of the pattern. + final RunAutomaton backwardAutomaton = inverseAutomatons.get(this.matchingPatterns[patternIndex]); + int state = backwardAutomaton.getInitialState(); + for (int pos = this.currentPosition - 1; pos >= 0; pos--) { + final char c = this.seq.charAt(pos); + state = backwardAutomaton.step(state, c); + if (state == -1) { + break; + } + if (backwardAutomaton.isAccept(state)) { + this.matchingPatternsStart[patternIndex] = pos; + } + } + } + + return this.matchingPatternsStart[patternIndex]; } public int end() { - return this.end; + return end(0); + } + + public int end(int patternIndex) { + if (this.matchingPatterns == null) { + return -1; + } + + if (this.matchingPatternsEnd[patternIndex] == 0) { + final int seqLength = this.seq.length(); + final int start = start(patternIndex); + // we go forward again using the forward automaton to find the end of the pattern. + final RunAutomaton forwardAutomaton = individualAutomatons.get(this.matchingPatterns[patternIndex]); + int state = forwardAutomaton.getInitialState(); + for (int pos = start; pos < seqLength; pos++) { + final char c = this.seq.charAt(pos); + state = forwardAutomaton.step(state, c); + if (state == -1) { + break; + } + if (forwardAutomaton.isAccept(state)) { + this.matchingPatternsEnd[patternIndex] = pos + 1; + } + } + } + + return this.matchingPatternsEnd[patternIndex]; } public int match() { - return this.matchingPattern; + return match(0); + } + + public int match(int patternIndex) { + return this.matchingPatterns == null ? -1: this.matchingPatterns[patternIndex]; + } + + public String pattern() { + return pattern(0); + } + + public String pattern(int patternIndex) { + final RunAutomaton automaton = individualAutomatons.get(this.matchingPatterns[patternIndex]); + return automaton.toString(); + } + + public int[] matches() { + return this.matchingPatterns; } public boolean found() { - return this.matchingPattern >= 0; + return this.matchingPatterns != null; } @@ -124,58 +192,29 @@ public boolean found() { * If no match is found the function return false. */ public boolean next() { - this.start = -1; - this.matchingPattern = -1; + this.matchingPatterns = null; + this.matchingPatternsStart = null; + this.matchingPatternsEnd = null; final int seqLength = this.seq.length(); - { // first find a match and "choose the pattern". - int state = 0; - for (int pos=this.end; pos < seqLength; pos++) { - final char c = this.seq.charAt(pos); - state = automaton.step(state, c); - if (automaton.atLeastOneAccept[state]) { - // We found a match! - this.matchingPattern = automaton.accept[state][0]; - this.end = pos; - break; - } - } - if (this.matchingPattern == -1) { - return false; - } - } - { // we rewind using the backward automaton to find the start of the pattern. - final RunAutomaton backwardAutomaton = inverseAutomatons.get(this.matchingPattern); - int state = backwardAutomaton.getInitialState(); - for (int pos = this.end; pos >= 0; pos--) { - final char c = this.seq.charAt(pos); - state = backwardAutomaton.step(state, c); - if (state == -1) { - break; - } - if (backwardAutomaton.isAccept(state)) { - start = pos; - } - } - } - - { // we go forward again using the forward automaton to find the end of the pattern. - final RunAutomaton forwardAutomaton = individualAutomatons.get(this.matchingPattern); - int state = forwardAutomaton.getInitialState(); - for (int pos = this.start; pos < seqLength; pos++) { - final char c = this.seq.charAt(pos); - state = forwardAutomaton.step(state, c); - if (state == -1) { - break; - } - if (forwardAutomaton.isAccept(state)) { - this.end = pos + 1; + // first find a match and "choose the pattern". + int state = 0; + for (int pos = this.currentPosition; pos < seqLength; pos++) { + final char c = this.seq.charAt(pos); + state = automaton.step(state, c); + if (automaton.atLeastOneAccept[state]) { + // We found a match! + this.matchingPatterns = automaton.accept[state]; + this.matchingPatternsStart = new int[this.matchingPatterns.length]; + this.matchingPatternsEnd = new int[this.matchingPatterns.length]; + for (int i = 0; i < this.matchingPatterns.length; i++) { + this.matchingPatternsStart[i] = -1; } + this.currentPosition = pos + 1; + break; } } - return true; + return this.matchingPatterns != null; } - - } } From 24af33a53ff5eca58a108fe4c5bb4b3fb1d11182 Mon Sep 17 00:00:00 2001 From: nsi Date: Mon, 18 Apr 2016 14:48:43 +0100 Subject: [PATCH 10/14] Optimisation Switch allocation of start/end arrays out of next() method to start(), end() methods. --- .../multiregexp/MultiPatternSearcher.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java index 11f88b6..f7aaa3a 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java @@ -94,6 +94,11 @@ public int start() { public int start(int patternIndex) { if (this.matchingPatterns == null) { return -1; + } else if (this.matchingPatternsStart == null) { + this.matchingPatternsStart = new int[this.matchingPatterns.length]; + for (int i = 0; i < this.matchingPatterns.length; i++) { + this.matchingPatternsStart[i] = -1; + } } if (this.matchingPatternsStart[patternIndex] == -1) { @@ -122,6 +127,8 @@ public int end() { public int end(int patternIndex) { if (this.matchingPatterns == null) { return -1; + } else if (this.matchingPatternsEnd == null) { + this.matchingPatternsEnd = new int[this.matchingPatterns.length]; } if (this.matchingPatternsEnd[patternIndex] == 0) { @@ -197,18 +204,12 @@ public boolean next() { this.matchingPatternsEnd = null; final int seqLength = this.seq.length(); // first find a match and "choose the pattern". - int state = 0; - for (int pos = this.currentPosition; pos < seqLength; pos++) { + for (int state = 0, pos = this.currentPosition; pos < seqLength; pos++) { final char c = this.seq.charAt(pos); state = automaton.step(state, c); if (automaton.atLeastOneAccept[state]) { // We found a match! this.matchingPatterns = automaton.accept[state]; - this.matchingPatternsStart = new int[this.matchingPatterns.length]; - this.matchingPatternsEnd = new int[this.matchingPatterns.length]; - for (int i = 0; i < this.matchingPatterns.length; i++) { - this.matchingPatternsStart[i] = -1; - } this.currentPosition = pos + 1; break; } From bf6e7e4fb8c2e6d3b7b6e3db66faaccd6aa33aed Mon Sep 17 00:00:00 2001 From: nsi Date: Tue, 19 Apr 2016 11:16:38 +0100 Subject: [PATCH 11/14] Optimisation Initialise maps and lists with known size --- .../com/fulmicoton/multiregexp/MultiPatternSearcher.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java index f7aaa3a..f8413e2 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPatternSearcher.java @@ -30,11 +30,11 @@ public class MultiPatternSearcher final List individualAutomatons, boolean tableize) { this.automaton = automaton; - this.individualAutomatons = new ArrayList<>(); + this.individualAutomatons = new ArrayList<>(individualAutomatons.size()); for (final Automaton individualAutomaton : individualAutomatons) { this.individualAutomatons.add(new RunAutomaton(individualAutomaton, tableize)); } - this.inverseAutomatons = new ArrayList<>(this.individualAutomatons.size()); + this.inverseAutomatons = new ArrayList<>(individualAutomatons.size()); for (final Automaton individualAutomaton : individualAutomatons) { final Automaton inverseAutomaton = inverseAutomaton(individualAutomaton); this.inverseAutomatons.add(new RunAutomaton(inverseAutomaton, tableize)); @@ -42,7 +42,7 @@ public class MultiPatternSearcher } static Automaton inverseAutomaton(final Automaton automaton) { - final Map stateMapping = new HashMap<>(); + final Map stateMapping = new HashMap<>(automaton.getStates().size()); for (final State state : automaton.getStates()) { stateMapping.put(state, new State()); } @@ -57,7 +57,7 @@ static Automaton inverseAutomaton(final Automaton automaton) { stateMapping.get(automaton.getInitialState()).setAccept(true); final State initialState = new State(); inverseAutomaton.setInitialState(initialState); - final List epsilons = new ArrayList<>(); + final List epsilons = new ArrayList<>(automaton.getAcceptStates().size()); for (final State acceptState : automaton.getAcceptStates()) { final State invOrigState = stateMapping.get(acceptState); final StatePair statePair = new StatePair(initialState, invOrigState); From 1f2f2072014ae98da5d2fb9bea6946fa7a46b754 Mon Sep 17 00:00:00 2001 From: nsi Date: Tue, 19 Apr 2016 12:07:52 +0100 Subject: [PATCH 12/14] Add exceptions to makeAutomatonWithPrefix() so that patterns starting with ".*" and "^" do not have prefix attached. --- .../fulmicoton/multiregexp/MultiPattern.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java index 86e372b..4fe7e15 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java @@ -23,10 +23,17 @@ public static MultiPattern of(String... patterns) { return new MultiPattern(Arrays.asList(patterns)); } - public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix) { + public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix, String... exceptions) { final List automata = new ArrayList<>(); - for (final String ptn: this.patterns) { - final String prefixedPattern = prefix + ptn; + for (final String ptn : this.patterns) { + boolean addPrefix = true; + for (String exception : exceptions) { + if (ptn.startsWith(exception)) { + addPrefix = false; + break; + } + } + final String prefixedPattern = (addPrefix ? prefix: "") + ptn; final Automaton automaton = new RegExp(prefixedPattern).toAutomaton(); automaton.minimize(); automata.add(automaton); @@ -43,16 +50,15 @@ public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix) { * * @return A searcher object */ - public MultiPatternSearcher searcher() - { + public MultiPatternSearcher searcher() { return searcher(true); } public MultiPatternSearcher searcher(final boolean tableize) { - final MultiPatternAutomaton searcherAutomaton = makeAutomatonWithPrefix(".*"); + final MultiPatternAutomaton searcherAutomaton = makeAutomatonWithPrefix(".*", ".*", "^"); final List individualAutomatons = new ArrayList<>(); - for (final String pattern: this.patterns) { + for (final String pattern : this.patterns) { final Automaton automaton = new RegExp(pattern).toAutomaton(); automaton.minimize(); automaton.determinize(); From f612e5cbfbc5af57a58d1882abbb5712fd02ea1a Mon Sep 17 00:00:00 2001 From: nsi Date: Tue, 19 Apr 2016 12:30:59 +0100 Subject: [PATCH 13/14] Trivial - comment change version to 0.5.1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 585aeb5..519752c 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Add the following lines in the dependencies section of your `pom.xml` file. com.fulmicoton multiregexp - 0.3 + 0.5.1 ``` From b9a11f6f0487ba01d3bc1af4ad398464ffeccf0b Mon Sep 17 00:00:00 2001 From: nsi Date: Tue, 19 Apr 2016 12:56:38 +0100 Subject: [PATCH 14/14] Optimisation - initialise maps and lists with known size --- src/main/java/com/fulmicoton/multiregexp/MultiPattern.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java index 4fe7e15..0012c71 100644 --- a/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java +++ b/src/main/java/com/fulmicoton/multiregexp/MultiPattern.java @@ -24,7 +24,7 @@ public static MultiPattern of(String... patterns) { } public MultiPatternAutomaton makeAutomatonWithPrefix(String prefix, String... exceptions) { - final List automata = new ArrayList<>(); + final List automata = new ArrayList<>(this.patterns.size()); for (final String ptn : this.patterns) { boolean addPrefix = true; for (String exception : exceptions) { @@ -57,7 +57,7 @@ public MultiPatternSearcher searcher() { public MultiPatternSearcher searcher(final boolean tableize) { final MultiPatternAutomaton searcherAutomaton = makeAutomatonWithPrefix(".*", ".*", "^"); - final List individualAutomatons = new ArrayList<>(); + final List individualAutomatons = new ArrayList<>(this.patterns.size()); for (final String pattern : this.patterns) { final Automaton automaton = new RegExp(pattern).toAutomaton(); automaton.minimize();