diff --git a/configure.ac b/configure.ac
index 659a5cd976..fe726e760a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -14,7 +14,7 @@
# along with this library. If not, see
+ * Each element in the non-{@code null} but possibly empty array returned
+ * corresponds to one time series for which some data points have been
+ * matched by the query.
+ * @throws HBaseException if there was a problem communicating with HBase to
+ * perform the search.
+ */
+ @Override
+ public DataPoints[] run() throws HBaseException {
+ try {
+ return runAsync().joinUninterruptibly();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Should never be here", e);
+ }
+ }
+
+ /**
+ * Runs this query.
+ *
+ * @return The data points matched by this query and applied with percentile calculation
+ *
+ * Each element in the non-{@code null} but possibly empty array returned
+ * corresponds to one time series for which some data points have been
+ * matched by the query.
+ * @throws HBaseException if there was a problem communicating with HBase to
+ * perform the search.
+ * @throws IllegalStateException if the query is not a histogram query
+ */
+ @Override
+ public DataPoints[] runHistogram() throws HBaseException {
+ if (!isHistogramQuery()) {
+ throw new RuntimeException("Should never be here");
+ }
+
+ try {
+ return runHistogramAsync().joinUninterruptibly();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Should never be here", e);
+ }
+ }
+}
diff --git a/src/core/AbstractSpanGroup.java b/src/core/AbstractSpanGroup.java
new file mode 100644
index 0000000000..f9605a6b05
--- /dev/null
+++ b/src/core/AbstractSpanGroup.java
@@ -0,0 +1,70 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2010-2012 The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version. This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details. You should have received a copy
+// of the GNU Lesser General Public License along with this program. If not,
+// see
+ * This is used for queries to the TSDB, where we might group multiple
+ * {@link Span}s that are for the same time series but different tags
+ * together. We need to "hide" data points that are outside of the
+ * time period of the query and do on-the-fly aggregation of the data
+ * points coming from the different Spans, using an {@link Aggregator}.
+ * Since not all the Spans will have their data points at exactly the
+ * same time, we also do on-the-fly linear interpolation. If needed,
+ * this view can also return the rate of change instead of the actual
+ * data points.
+ *
+ * This is one of the rare (if not the only) implementations of
+ * {@link DataPoints} for which {@link #getTags} can potentially return
+ * an empty map.
+ *
+ * The implementation can also dynamically downsample the data when a
+ * sampling interval a downsampling function (in the form of an
+ * {@link Aggregator}) are given. This is done by using a special
+ * iterator when using the {@link Span.DownsamplingIterator}.
+ */
+abstract class AbstractSpanGroup implements DataPoints {
+ /**
+ * Finds the {@code i}th data point of this group in {@code O(n)}.
+ * Where {@code n} is the number of data points in this group.
+ */
+ protected DataPoint getDataPoint(int i) {
+ if (i < 0) {
+ throw new IndexOutOfBoundsException("negative index: " + i);
+ }
+ final int saved_i = i;
+ final SeekableView it = iterator();
+ DataPoint dp = null;
+ while (it.hasNext() && i >= 0) {
+ dp = it.next();
+ i--;
+ }
+ if (i != -1 || dp == null) {
+ throw new IndexOutOfBoundsException("index " + saved_i
+ + " too large (it's >= " + size() + ") for " + this);
+ }
+ return dp;
+ }
+}
diff --git a/src/core/AggregationIterator.java b/src/core/AggregationIterator.java
index a5d0f541fa..7e2210c721 100644
--- a/src/core/AggregationIterator.java
+++ b/src/core/AggregationIterator.java
@@ -783,7 +783,7 @@ public double nextDoubleValue() {
r = Double.MAX_VALUE;
break;
case MIN:
- r = Double.MIN_VALUE;
+ r = -Double.MAX_VALUE;
break;
case PREV:
r = y0;
diff --git a/src/core/ByteBufferList.java b/src/core/ByteBufferList.java
index 443750e082..9f744bfefc 100644
--- a/src/core/ByteBufferList.java
+++ b/src/core/ByteBufferList.java
@@ -58,6 +58,20 @@ public void add(final byte[] buf, final int offset, final int len) {
total_length += len;
}
+ /**
+ * Removes the last added segment from the segments array and returns it to the caller
+ *
+ * @return byte array representing the most recently added segment or null if no segments exist
+ */
+ public BufferSegment removeLastSegment() {
+ if (segments.isEmpty()) {
+ return null;
+ }
+ BufferSegment seg = segments.remove(segments.size() - 1);
+ total_length -= seg.len;
+ return seg;
+ }
+
/**
* Get the most recently added segment.
*
@@ -73,7 +87,7 @@ public byte[] getLastSegment() {
/**
* Get the number of segments that have added to this buffer list.
- *
+ *
* @return the segment count
*/
public int segmentCount() {
@@ -82,7 +96,7 @@ public int segmentCount() {
/**
* Get the accumulated bytes as a single byte array (may be a zero-byte array if empty).
- *
+ *
* @param padding the number of additional bytes to include at the end
* @return the accumulated bytes
*/
diff --git a/src/core/ColumnDatapointIterator.java b/src/core/ColumnDatapointIterator.java
index 8df8a20626..0d1858678b 100644
--- a/src/core/ColumnDatapointIterator.java
+++ b/src/core/ColumnDatapointIterator.java
@@ -120,6 +120,19 @@ public void writeToBuffers(ByteBufferList compQualifier, ByteBufferList compValu
compValue.add(value, value_offset, current_val_length);
}
+ /**
+ * Write a new qualifier and its associated value to the compacted qualifier buffer and compacted values buffer respectively.
+ *
+ * @param newQualifier - the new qualifier
+ * @param newVal - the new value
+ * @param compactedQual - the qualifiers buffer
+ * @param compactedVal - the values buffer
+ */
+ public void writeToBuffers(byte[] newQualifier, byte[] newVal, ByteBufferList compactedQual, ByteBufferList compactedVal) {
+ compactedQual.add(newQualifier, 0, newQualifier.length);
+ compactedVal.add(newVal, 0, newVal.length);
+ }
+
public void writeToBuffersFromOffset(ByteBufferList compQualifier, ByteBufferList compValue, Pair offsets, Pair offsetLengths) {
compQualifier.add(qualifier, offsets.getKey(), offsetLengths.getKey());
compValue.add(value, offsets.getValue(), offsetLengths.getValue());
@@ -186,6 +199,20 @@ private boolean update() {
return true;
}
+ /**
+ * @return the flags mask from the qualifier
+ */
+ public short getFlagsFromQualifier() {
+ return Internal.getFlagsFromQualifier(qualifier, qualifier_offset);
+ }
+
+ /**
+ * @return the column timestamp
+ */
+ public long getColumnTimestamp() {
+ return column_timestamp;
+ }
+
// order in ascending order by timestamp, descending order by row timestamp (so we find the
// entry we are going to keep first, and don't have to copy over it)
@Override
diff --git a/src/core/CompactionQueue.java b/src/core/CompactionQueue.java
index 857d60fbd2..859197bbdd 100644
--- a/src/core/CompactionQueue.java
+++ b/src/core/CompactionQueue.java
@@ -558,7 +558,7 @@ private void defaultMergeDataPoints(ByteBufferList compacted_qual,
final byte[] discardedVal = col.getCopyOfCurrentValue();
if (!Arrays.equals(existingVal, discardedVal)) {
duplicates_different.incrementAndGet();
- if (!tsdb.config.fix_duplicates()) {
+ if (!tsdb.config.fix_duplicates() && !tsdb.config.sum_duplicates()) {
throw new IllegalDataException("Duplicate timestamp for key="
+ Arrays.toString(row.get(0).key()) + ", ms_offset=" + ts + ", older="
+ Arrays.toString(existingVal) + ", newer=" + Arrays.toString(discardedVal)
@@ -570,6 +570,53 @@ private void defaultMergeDataPoints(ByteBufferList compacted_qual,
} else {
duplicates_same.incrementAndGet();
}
+
+ if (tsdb.config.sum_duplicates()) {
+ short current_flags = Internal.getFlagsFromQualifier(compacted_qual.getLastSegment());
+ short discarded_flags = col.getFlagsFromQualifier();
+
+ boolean is_current_long = ((current_flags & Const.FLAG_FLOAT) == 0x0);
+ boolean is_discarded_long = ((discarded_flags & Const.FLAG_FLOAT) == 0x0);
+
+ /* the current value flags determine the type of the output (long or double) */
+
+ double current_val = 0.0;
+ double discarded_val = 0.0;
+
+ if (is_current_long) {
+ current_val = Internal.extractIntegerValue(existingVal, 0, (byte)current_flags);
+ } else {
+ current_val = Internal.extractFloatingPointValue(existingVal, 0, (byte) current_flags);
+ }
+
+ if (is_discarded_long) {
+ discarded_val = Internal.extractIntegerValue(discardedVal, 0, (byte)discarded_flags);
+ } else {
+ discarded_val = Internal.extractFloatingPointValue(discardedVal, 0, (byte) current_flags);
+ }
+
+ current_val += discarded_val;
+ byte[] new_val = null;
+ current_flags = 0;
+
+ if (is_current_long) {
+ new_val = Bytes.fromLong((long) current_val);
+ current_flags = (short) (new_val.length - 1);
+ } else {
+ new_val = Bytes.fromLong(Double.doubleToRawLongBits(current_val));
+ current_flags = Const.FLAG_FLOAT | 0x7;
+ }
+
+ final byte[] new_qualifier = Internal.buildQualifier(col.getColumnTimestamp(), current_flags);
+
+ /* now remove the last value & qualifier (existing one) and write the updated value and qualifier */
+
+ prevTs = ts;
+ compacted_val.removeLastSegment();
+ compacted_qual.removeLastSegment();
+ col.writeToBuffers(new_qualifier, new_val, compacted_qual, compacted_val);
+ ms_in_row |= col.isMilliseconds();
+ s_in_row |= !col.isMilliseconds();
} else {
prevTs = ts;
col.writeToBuffers(compacted_qual, compacted_val);
diff --git a/src/core/Downsampler.java b/src/core/Downsampler.java
index 173ef7e65b..97765a8d70 100644
--- a/src/core/Downsampler.java
+++ b/src/core/Downsampler.java
@@ -213,10 +213,7 @@ public double nextDoubleValue() {
specification.getFunction() == Aggregators.COUNT) {
double count = 0;
while (values_in_interval.hasNextValue()) {
- count += values_in_interval.nextValueCount();
- // WARNING: consume and move next or we'll be stuck in an infinite
- // loop here.
- values_in_interval.nextDoubleValue();
+ count += values_in_interval.nextDoubleValue();
}
value = count;
} else {
diff --git a/src/core/FillingDownsampler.java b/src/core/FillingDownsampler.java
index 273f0ed18b..5edf23509b 100644
--- a/src/core/FillingDownsampler.java
+++ b/src/core/FillingDownsampler.java
@@ -244,10 +244,7 @@ public double nextDoubleValue() {
specification.getFunction() == Aggregators.COUNT) {
double count = 0;
while (values_in_interval.hasNextValue()) {
- count += values_in_interval.nextValueCount();
- // WARNING: consume and move next or we'll be stuck in an infinite
- // loop here.
- values_in_interval.nextDoubleValue();
+ count += values_in_interval.nextDoubleValue();
}
value = count;
} else {
diff --git a/src/core/GroupCallback.java b/src/core/GroupCallback.java
new file mode 100644
index 0000000000..da835dbd63
--- /dev/null
+++ b/src/core/GroupCallback.java
@@ -0,0 +1,30 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2010-2012 The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version. This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details. You should have received a copy
+// of the GNU Lesser General Public License along with this program. If not,
+// see > {
+ /**
+ * We're only waiting for all callbacks to complete, ignoring their return values.
+ *
+ * @param ignored The return values of the individual callbacks - ignored
+ * @return null
+ */
+ @Override
+ public Object call(ArrayList ignored) {
+ return null;
+ }
+}
diff --git a/src/core/IncomingDataPoints.java b/src/core/IncomingDataPoints.java
index 0456e47be2..94795d74de 100644
--- a/src/core/IncomingDataPoints.java
+++ b/src/core/IncomingDataPoints.java
@@ -18,6 +18,7 @@
import java.util.Date;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
@@ -45,6 +46,11 @@ final class IncomingDataPoints implements WritableDataPoints {
*/
static final Histogram putlatency = new Histogram(16000, (short) 2, 100);
+ /**
+ * Keep track of the number of UIDs that came back null with auto_metric disabled.
+ */
+ static final AtomicLong auto_metric_rejection_count = new AtomicLong();
+
/** The {@code TSDB} instance we belong to. */
private final TSDB tsdb;
@@ -137,9 +143,14 @@ static byte[] rowKeyTemplate(final TSDB tsdb, final String metric,
short pos = (short) Const.SALT_WIDTH();
- copyInRowKey(row, pos,
- (tsdb.config.auto_metric() ? tsdb.metrics.getOrCreateId(metric)
- : tsdb.metrics.getId(metric)));
+ byte[] metric_id = (tsdb.config.auto_metric() ? tsdb.metrics.getOrCreateId(metric)
+ : tsdb.metrics.getId(metric));
+
+ if(!tsdb.config.auto_metric() && metric_id == null) {
+ auto_metric_rejection_count.incrementAndGet();
+ }
+
+ copyInRowKey(row, pos, metric_id);
pos += metric_width;
pos += Const.TIMESTAMP_BYTES;
@@ -151,60 +162,6 @@ static byte[] rowKeyTemplate(final TSDB tsdb, final String metric,
return row;
}
- /**
- * Returns a partially initialized row key for this metric and these tags. The
- * only thing left to fill in is the base timestamp.
- *
- * @since 2.0
- */
- static Deferred rowKeyTemplateAsync(final TSDB tsdb,
- final String metric, final Map tags) {
- final short metric_width = tsdb.metrics.width();
- final short tag_name_width = tsdb.tag_names.width();
- final short tag_value_width = tsdb.tag_values.width();
- final short num_tags = (short) tags.size();
-
- int row_size = (Const.SALT_WIDTH() + metric_width + Const.TIMESTAMP_BYTES
- + tag_name_width * num_tags + tag_value_width * num_tags);
- final byte[] row = new byte[row_size];
-
- // Lookup or create the metric ID.
- final Deferred metric_id;
- if (tsdb.config.auto_metric()) {
- metric_id = tsdb.metrics.getOrCreateIdAsync(metric, metric, tags);
- } else {
- metric_id = tsdb.metrics.getIdAsync(metric);
- }
-
- // Copy the metric ID at the beginning of the row key.
- class CopyMetricInRowKeyCB implements Callback {
- public byte[] call(final byte[] metricid) {
- copyInRowKey(row, (short) Const.SALT_WIDTH(), metricid);
- return row;
- }
- }
-
- // Copy the tag IDs in the row key.
- class CopyTagsInRowKeyCB implements
- Callback, ArrayList> {
- public Deferred call(final ArrayList tags) {
- short pos = (short) (Const.SALT_WIDTH() + metric_width);
- pos += Const.TIMESTAMP_BYTES;
- for (final byte[] tag : tags) {
- copyInRowKey(row, pos, tag);
- pos += tag.length;
- }
- // Once we've resolved all the tags, schedule the copy of the metric
- // ID and return the row key we produced.
- return metric_id.addCallback(new CopyMetricInRowKeyCB());
- }
- }
-
- // Kick off the resolution of all tags.
- return Tags.resolveOrCreateAllAsync(tsdb, metric, tags)
- .addCallbackDeferring(new CopyTagsInRowKeyCB());
- }
-
public void setSeries(final String metric, final Map tags) {
checkMetricAndTags(metric, tags);
try {
@@ -355,12 +312,18 @@ public Deferred call(final Boolean allowed) throws Exception {
point.setDurable(!batch_import);
return tsdb.client.append(point);/* .addBoth(cb) */
} else {
- final PutRequest point = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.table, row, TSDB.FAMILY,
- qualifier, value, timestamp);
+ boolean isLong = ((flags & Const.FLAG_FLOAT) == 0x0);
+ if (isLong && tsdb.config.use_hbase_counters()) {
+ AtomicIncrementRequest counterRequest = new AtomicIncrementRequest(tsdb.table, row, TSDB.FAMILY, qualifier, Bytes.getLong(value));
+ return tsdb.client.atomicIncrement(counterRequest, !batch_import);
+ } else {
+ final PutRequest point = new PutRequest(tsdb.table, row, TSDB.FAMILY,
+ qualifier, value);
point.setDurable(!batch_import);
return tsdb.client.put(point)/* .addBoth(cb) */;
}
}
+ }
@Override
public String toString() {
return "IncomingDataPoints.addPointInternal Write Callback";
@@ -391,6 +354,7 @@ private long baseTime() {
public Deferred addPoint(final long timestamp, final long value) {
final byte[] v;
+
if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) {
v = new byte[] { (byte) value };
} else if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) {
diff --git a/src/core/Internal.java b/src/core/Internal.java
index 3d1b82171e..146a272c78 100644
--- a/src/core/Internal.java
+++ b/src/core/Internal.java
@@ -241,30 +241,51 @@ public static ArrayList extractDataPoints(final ArrayList row,
final byte[] qual = kv.qualifier();
final int len = qual.length;
final byte[] val = kv.value();
-
- if (len % 2 != 0) {
- // skip a non data point column
- continue;
- } else if (len == 2) { // Single-value cell.
- // Maybe we need to fix the flags in the qualifier.
- final byte[] actual_val = fixFloatingPointValue(qual[1], val);
- final byte q = fixQualifierFlags(qual[1], actual_val.length);
- final byte[] actual_qual;
-
- if (q != qual[1]) { // We need to fix the qualifier.
- actual_qual = new byte[] { qual[0], q }; // So make a copy.
- } else {
- actual_qual = qual; // Otherwise use the one we already have.
+
+ // when enable_appends set to true, should get qualifier and value from the HBase Column Value
+ if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) {
+ int idx = 0;
+ int q_length = 0;
+ int v_length = 0;
+ while (idx < kv.value().length) {
+ q_length = Internal.getQualifierLength(kv.value(), idx);
+ v_length = Internal.getValueLengthFromQualifier(kv.value(), idx);
+ final byte[] q = new byte[q_length];
+ final byte[] v = new byte[v_length];
+ System.arraycopy(kv.value(),idx,q,0,q_length);
+ System.arraycopy(kv.value(),idx + q_length,v, 0, v_length);
+ idx += q_length + v_length;
+
+ final Cell cell = new Cell(q, v);
+ cells.add(cell);
}
-
- final Cell cell = new Cell(actual_qual, actual_val);
- cells.add(cell);
- continue;
- } else if (len == 4 && inMilliseconds(qual[0])) {
- // since ms support is new, there's nothing to fix
- final Cell cell = new Cell(qual, val);
- cells.add(cell);
continue;
+ } else {
+
+ if (len % 2 != 0) {
+ // skip a non data point column
+ continue;
+ } else if (len == 2) { // Single-value cell.
+ // Maybe we need to fix the flags in the qualifier.
+ final byte[] actual_val = fixFloatingPointValue(qual[1], val);
+ final byte q = fixQualifierFlags(qual[1], actual_val.length);
+ final byte[] actual_qual;
+
+ if (q != qual[1]) { // We need to fix the qualifier.
+ actual_qual = new byte[]{qual[0], q}; // So make a copy.
+ } else {
+ actual_qual = qual; // Otherwise use the one we already have.
+ }
+
+ final Cell cell = new Cell(actual_qual, actual_val);
+ cells.add(cell);
+ continue;
+ } else if (len == 4 && inMilliseconds(qual[0])) {
+ // since ms support is new, there's nothing to fix
+ final Cell cell = new Cell(qual, val);
+ cells.add(cell);
+ continue;
+ }
}
// Now break it down into Cells.
diff --git a/src/core/MultiGetQuery.java b/src/core/MultiGetQuery.java
index d12f4afd7e..830a68b5e7 100644
--- a/src/core/MultiGetQuery.java
+++ b/src/core/MultiGetQuery.java
@@ -20,6 +20,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.SortedMap;
import java.util.TreeMap;
import java.util.AbstractMap.SimpleEntry;
import java.util.concurrent.ConcurrentHashMap;
@@ -89,11 +90,11 @@ public class MultiGetQuery {
private final Map>>> histMap = Maps.newConcurrentMap();
- private final Deferred> results =
- new Deferred>();
+ private final Deferred> results =
+ new Deferred>();
- private final Deferred> histogramResults =
- new Deferred>();
+ private final Deferred> histogramResults =
+ new Deferred>();
private final ArrayList> multi_get_tasks;
private final ArrayList multi_get_indexs;
@@ -608,7 +609,7 @@ void close(final boolean ok) {
* Initiate the get requests and return the tree map of results.
* @return A non-null tree map of results (may be empty)
*/
- public Deferred> fetch() {
+ public Deferred> fetch() {
if(tags.isEmpty()) {
return Deferred.fromResult(null);
}
@@ -620,7 +621,7 @@ public Deferred> fetch() {
* Initiate the get requests and return the tree map of results.
* @return A non-null tree map of results (may be empty)
*/
- public Deferred> fetchHistogram() {
+ public Deferred> fetchHistogram() {
startFetch();
return histogramResults;
}
diff --git a/src/core/Query.java b/src/core/Query.java
index 3d95834004..5cec6c7c80 100644
--- a/src/core/Query.java
+++ b/src/core/Query.java
@@ -171,7 +171,23 @@ public void setTimeSeries(final List tsuids,
*/
public Deferred configureFromQuery(final TSQuery query,
final int index);
-
+
+ /**
+ * Prepares a query against HBase by setting up group bys and resolving
+ * strings to UIDs asynchronously. This replaces calls to all of the setters
+ * like the {@link setTimeSeries}, {@link setStartTime}, etc.
+ * Make sure to wait on the deferred return before calling {@link runAsync}.
+ * @param query The main query to fetch the start and end time from
+ * @param index The index of which sub query we're executing
+ * @param force_raw If true, always get the data from the raw table; disables rollups
+ * @throws IllegalArgumentException if the query was missing sub queries or
+ * the index was out of bounds.
+ * @throws NoSuchUniqueName if the name of a metric, or a tag name/value
+ * does not exist. (Bubbles up through the deferred)
+ * @since 2.4
+ */
+ Deferred configureFromQuery(final TSQuery query, final int index, boolean force_raw);
+
/**
* Downsamples the results by specifying a fixed interval between points.
*
@@ -181,7 +197,7 @@ public Deferred configureFromQuery(final TSQuery query,
* way we get this one data point is by aggregating all the data points of
* that interval together using an {@link Aggregator}. This enables you
* to compute things like the 5-minute average or 10 minute 99th percentile.
- * @param interval Number of seconds wanted between each data point.
+ * @param interval Number of milliseconds wanted between each data point.
* @param downsampler Aggregation function to use to group data points
* within an interval.
*/
@@ -262,7 +278,19 @@ public Deferred configureFromQuery(final TSQuery query,
* @return
*/
public boolean isHistogramQuery();
-
+
+ /**
+ * @return Whether or not this is a rollup query
+ * @since 2.4
+ */
+ public boolean isRollupQuery();
+
+ /**
+ * @return whether this query needs to be split.
+ * @since 2.4
+ */
+ public boolean needsSplitting();
+
/**
* Set the percentile calculation parameters for this query if this is
* a histogram query
diff --git a/src/core/RpcResponder.java b/src/core/RpcResponder.java
new file mode 100644
index 0000000000..97e7b22bb8
--- /dev/null
+++ b/src/core/RpcResponder.java
@@ -0,0 +1,110 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2010-2017 The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version. This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details. You should have received a copy
+// of the GNU Lesser General Public License along with this program. If not,
+// see spans;
+ private final SortedMap spans;
- private final TreeMap histSpans;
+ private final SortedMap histSpans;
/** The list of pre-configured scanners. One scanner should be created per
* salt bucket. */
@@ -93,11 +95,11 @@ public class SaltScanner {
List>>>();
/** A deferred to call with the spans on completion */
- private final Deferred> results =
- new Deferred>();
+ private final Deferred> results =
+ new Deferred>();
- private final Deferred> histogramResults =
- new Deferred>();
+ private final Deferred> histogramResults =
+ new Deferred>();
/** The metric this scanner set is dealing with. If a row comes in with a
* different metric we toss an exception. This shouldn't happen though. */
@@ -124,7 +126,7 @@ public class SaltScanner {
private final long max_bytes;
/** A latch used to determine how many scanners are still running */
- private final CountDownLatch countdown;
+ private final AtomicInteger countdown;
/** When the scanning started. We store the scan latency once all scanners
* are done.*/
@@ -226,8 +228,8 @@ public SaltScanner(final TSDB tsdb, final byte[] metric,
}
this.scanners = scanners;
- this.spans = spans;
- this.histSpans = histogramSpans;
+ this.spans = spans != null ? Collections.synchronizedSortedMap(spans) : null;
+ this.histSpans = histogramSpans != null ? Collections.synchronizedSortedMap(histogramSpans) : null;
this.metric = metric;
this.tsdb = tsdb;
this.filters = filters;
@@ -235,7 +237,7 @@ public SaltScanner(final TSDB tsdb, final byte[] metric,
this.rollup_query = rollup_query;
this.query_stats = query_stats;
this.query_index = query_index;
- countdown = new CountDownLatch(scanners.size());
+ countdown = new AtomicInteger(scanners.size());
if (rollup_query != null && RollupQuery.isValidQuery(rollup_query)) {
is_rollup = true;
if (rollup_query.getRollupAgg() == Aggregators.AVG) {
@@ -264,7 +266,7 @@ public SaltScanner(final TSDB tsdb, final byte[] metric,
* first error will be returned, others will be logged.
* @return A deferred to wait on for results.
*/
- public Deferred> scan() {
+ public Deferred> scan() {
start_time = System.currentTimeMillis();
int i = 0;
for (final Scanner scanner: scanners) {
@@ -273,7 +275,7 @@ public Deferred> scan() {
return results;
}
- public Deferred> scanHistogram() {
+ public Deferred> scanHistogram() {
start_time = DateTime.currentTimeMillis();
int index = 0;
@@ -462,7 +464,7 @@ final class ScannerCB implements Callback>> {
private final Scanner scanner;
private final int index;
- private final List kvs = new ArrayList();
+ private final List kvs = Collections.synchronizedList(new ArrayList());
private final ByteMap> annotations =
new ByteMap>();
private final Set skips = Collections.newSetFromMap(
@@ -489,7 +491,8 @@ final class ScannerCB implements Callback> rows)
@@ -552,7 +555,24 @@ public Object call(final ArrayList> rows)
final List> lookups =
filters != null && !filters.isEmpty() ?
new ArrayList>(rows.size()) : null;
-
+
+ // fail the query when the timeout exceeded
+ if (this.query_timeout > 0 && fetch_time > (this.query_timeout * 1000000)) {
+ try {
+ close(false);
+ handleException(
+ new QueryException(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE,
+ "Sorry, your query timed out. Time limit: "
+ + this.query_timeout + " ms, fetch time: "
+ + (double)(fetch_time)/1000000 + " ms. Please try filtering "
+ + "using more tags or decrease your time range."));
+ return false;
+ } catch (Exception e) {
+ LOG.error("Sorry, Scanner is closed: " + scanner, e);
+ return false;
+ }
+ }
+
// validation checking before processing the next set of results. It's
// kinda funky but we want to allow queries to sneak through that were
// just a *tad* over the limits so that's why we don't check at the
@@ -909,7 +929,7 @@ void close(final boolean ok) {
if (ok && exception == null) {
validateAndTriggerCallback(kvs, annotations, histograms);
} else {
- countdown.countDown();
+ countdown.decrementAndGet();
}
}
}
@@ -924,10 +944,9 @@ private void validateAndTriggerCallback(
final Map> annotations,
final List>> histograms) {
- countdown.countDown();
- final long count = countdown.getCount();
+ int scannersRunning = countdown.decrementAndGet();
if (kvs.size() > 0) {
- kv_map.put((int) count, kvs);
+ kv_map.put(scannersRunning, kvs);
}
for (final byte[] key : annotations.keySet()) {
@@ -939,10 +958,10 @@ private void validateAndTriggerCallback(
}
if (histograms.size() > 0) {
- histMap.put((int) count, histograms);
+ histMap.put(scannersRunning, histograms);
}
-
- if (countdown.getCount() <= 0) {
+
+ if (scannersRunning <= 0) {
try {
mergeAndReturnResults();
} catch (final Exception ex) {
@@ -960,7 +979,7 @@ private void validateAndTriggerCallback(
*/
private void handleException(final Exception e) {
// make sure only one scanner can set the exception
- countdown.countDown();
+ countdown.decrementAndGet();
if (exception == null) {
synchronized (this) {
if (exception == null) {
diff --git a/src/core/SeekableViewChain.java b/src/core/SeekableViewChain.java
new file mode 100644
index 0000000000..63c0454240
--- /dev/null
+++ b/src/core/SeekableViewChain.java
@@ -0,0 +1,97 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2014 The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version. This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details. You should have received a copy
+// of the GNU Lesser General Public License along with this program. If not,
+// see iterators;
+ private int currentIterator;
+
+ SeekableViewChain(List iterators) {
+ this.iterators = iterators;
+ }
+
+ /**
+ * Returns {@code true} if this view has more elements.
+ */
+ @Override
+ public boolean hasNext() {
+ SeekableView iterator = getCurrentIterator();
+ return iterator != null && iterator.hasNext();
+ }
+
+ /**
+ * Returns a view on the next data point.
+ * No new object gets created, the referenced returned is always the same
+ * and must not be stored since its internal data structure will change the
+ * next time {@code next()} is called.
+ *
+ * @throws NoSuchElementException if there were no more elements to iterate
+ * on (in which case {@link #hasNext} would have returned {@code false}.
+ */
+ @Override
+ public DataPoint next() {
+ SeekableView iterator = getCurrentIterator();
+ if (iterator == null || !iterator.hasNext()) {
+ throw new NoSuchElementException("No elements left in iterator");
+ }
+
+ DataPoint next = iterator.next();
+
+ if (!iterator.hasNext()) {
+ currentIterator++;
+ }
+
+ return next;
+ }
+
+ /**
+ * Unsupported operation.
+ *
+ * @throws UnsupportedOperationException always.
+ */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Removing items is not supported");
+ }
+
+ /**
+ * Advances the iterator to the given point in time.
+ *
+ * This allows the iterator to skip all the data points that are strictly
+ * before the given timestamp.
+ *
+ * @param timestamp A strictly positive 32 bit UNIX timestamp (in seconds).
+ * @throws IllegalArgumentException if the timestamp is zero, or negative,
+ * or doesn't fit on 32 bits (think "unsigned int" -- yay Java!).
+ */
+ @Override
+ public void seek(long timestamp) {
+ for (final SeekableView it : iterators) {
+ it.seek(timestamp);
+ }
+ }
+
+ private SeekableView getCurrentIterator() {
+ while (currentIterator < iterators.size()) {
+ if (iterators.get(currentIterator).hasNext()) {
+ return iterators.get(currentIterator);
+ }
+ currentIterator++;
+ }
+ return null;
+ }
+}
diff --git a/src/core/SpanGroup.java b/src/core/SpanGroup.java
index 94bed154d0..07beaaa5d5 100644
--- a/src/core/SpanGroup.java
+++ b/src/core/SpanGroup.java
@@ -12,14 +12,7 @@
// see annotations;
@@ -109,7 +102,10 @@ final class SpanGroup implements DataPoints {
/** The TSDB to which we belong, used for resolution */
private final TSDB tsdb;
-
+
+ /** The group we belong to */
+ private byte[] group;
+
/**
* Ctor.
* @param tsdb The TSDB we belong to.
@@ -231,7 +227,7 @@ final class SpanGroup implements DataPoints {
final long query_end,
final int query_index) {
this(tsdb, start_time, end_time, spans, rate, rate_options, aggregator,
- downsampler, query_start, query_end, query_index, null);
+ downsampler, query_start, query_end, query_index, null, new byte[0]);
}
/**
@@ -265,7 +261,8 @@ final class SpanGroup implements DataPoints {
final long query_start,
final long query_end,
final int query_index,
- final RollupQuery rollup_query) {
+ final RollupQuery rollup_query,
+ byte[] group) {
annotations = new ArrayList();
this.start_time = (start_time & Const.SECOND_MASK) == 0 ?
start_time * 1000 : start_time;
@@ -285,6 +282,7 @@ final class SpanGroup implements DataPoints {
this.query_index = query_index;
this.rollup_query = rollup_query;
this.tsdb = tsdb;
+ this.group = group;
}
/**
@@ -531,32 +529,22 @@ public SeekableView iterator() {
rate, rate_options, rollup_query);
}
- /**
- * Finds the {@code i}th data point of this group in {@code O(n)}.
- * Where {@code n} is the number of data points in this group.
- */
- private DataPoint getDataPoint(int i) {
- if (i < 0) {
- throw new IndexOutOfBoundsException("negative index: " + i);
- }
- final int saved_i = i;
- final SeekableView it = iterator();
- DataPoint dp = null;
- while (it.hasNext() && i >= 0) {
- dp = it.next();
- i--;
- }
- if (i != -1 || dp == null) {
- throw new IndexOutOfBoundsException("index " + saved_i
- + " too large (it's >= " + size() + ") for " + this);
- }
- return dp;
- }
-
public long timestamp(final int i) {
return getDataPoint(i).timestamp();
}
+ /**
+ * Returns the group the spans in here belong to.
+ *
+ * Returns null if the NONE aggregator was requested in the query
+ * Returns an empty array if there were no group bys and they're all in the same group
+ * Returns the group otherwise
+ * @return The group
+ */
+ public byte[] group() {
+ return group;
+ }
+
public boolean isInteger(final int i) {
return getDataPoint(i).isInteger();
}
@@ -585,7 +573,8 @@ private String toStringSharedAttributes() {
+ ", aggregator=" + aggregator
+ ", downsampler=" + downsampler
+ ", query_start=" + query_start
- + ", query_end" + query_end
+ + ", query_end=" + query_end
+ + ", group=" + Arrays.toString(group)
+ ')';
}
@@ -602,6 +591,10 @@ public boolean isPercentile() {
public float getPercentile() {
throw new UnsupportedOperationException("getPercentile not supported");
}
+
+ public List getSpans() {
+ return spans;
+ }
/**
* Resolves the set of tag keys to their string names.
diff --git a/src/core/SplitRollupQuery.java b/src/core/SplitRollupQuery.java
new file mode 100644
index 0000000000..7121c99762
--- /dev/null
+++ b/src/core/SplitRollupQuery.java
@@ -0,0 +1,480 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2010-2012 The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version. This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details. You should have received a copy
+// of the GNU Lesser General Public License along with this program. If not,
+// see rollupResolution;
+ private Deferred rawResolution;
+
+ SplitRollupQuery(final TSDB tsdb, TsdbQuery rollupQuery, Deferred rollupResolution) {
+ if (rollupQuery == null) {
+ throw new IllegalArgumentException("Rollup query cannot be null");
+ }
+
+ this.tsdb = tsdb;
+
+ this.rollupQuery = rollupQuery;
+ this.rollupResolution = rollupResolution;
+ }
+
+ /**
+ * Returns the start time of the graph.
+ *
+ * @return A strictly positive integer.
+ * @throws IllegalStateException if {@link #setStartTime(long)} was never
+ * called on this instance before.
+ */
+ @Override
+ public long getStartTime() {
+ return rollupQuery != null ? rollupQuery.getStartTime() : rawQuery.getStartTime();
+ }
+
+ /**
+ * Sets the start time of the graph. Converts the timestamp to milliseconds if necessary.
+ *
+ * @param timestamp The start time, all the data points returned will have a
+ * timestamp greater than or equal to this one.
+ * @throws IllegalArgumentException if timestamp is less than or equal to 0,
+ * or if it can't fit on 32 bits.
+ * @throws IllegalArgumentException if
+ * {@code timestamp >= }{@link #getEndTime getEndTime}.
+ */
+ @Override
+ public void setStartTime(long timestamp) {
+ if ((timestamp & Const.SECOND_MASK) == 0) { timestamp *= 1000L; }
+
+ if (rollupQuery == null) {
+ rawQuery.setStartTime(timestamp);
+ return;
+ }
+
+ if (rollupQuery.getEndTime() <= timestamp) {
+ rollupQuery = null;
+ rawQuery.setStartTime(timestamp);
+ return;
+ }
+
+ rollupQuery.setStartTime(timestamp);
+ }
+
+ /**
+ * Returns the end time of the graph.
+ *
+ * If {@link #setEndTime} was never called before, this method will
+ * automatically execute
+ * {@link #setEndTime setEndTime}{@code (System.currentTimeMillis() / 1000)}
+ * to set the end time.
+ *
+ * @return A strictly positive integer.
+ */
+ @Override
+ public long getEndTime() {
+ return rawQuery != null ? rawQuery.getEndTime() : rollupQuery.getEndTime();
+ }
+
+ /**
+ * Sets the end time of the graph. Converts the timestamp to milliseconds if necessary.
+ *
+ * @param timestamp The end time, all the data points returned will have a
+ * timestamp less than or equal to this one.
+ * @throws IllegalArgumentException if timestamp is less than or equal to 0,
+ * or if it can't fit on 32 bits.
+ * @throws IllegalArgumentException if
+ * {@code timestamp <= }{@link #getStartTime getStartTime}.
+ */
+ @Override
+ public void setEndTime(long timestamp) {
+ if ((timestamp & Const.SECOND_MASK) == 0) { timestamp *= 1000L; }
+
+ rawQuery.setEndTime(timestamp);
+ }
+
+ /**
+ * Returns whether or not the data queried will be deleted.
+ *
+ * @return A boolean
+ * @since 2.4
+ */
+ @Override
+ public boolean getDelete() {
+ return rawQuery.getDelete();
+ }
+
+ /**
+ * Sets whether or not the data queried will be deleted.
+ *
+ * @param delete True if data should be deleted, false otherwise.
+ * @since 2.4
+ */
+ @Override
+ public void setDelete(boolean delete) {
+ if (rollupQuery != null) {
+ rollupQuery.setDelete(delete);
+ }
+ rawQuery.setDelete(delete);
+ }
+
+ /**
+ * Sets the time series to the query.
+ *
+ * @param metric The metric to retrieve from the TSDB.
+ * @param tags The set of tags of interest.
+ * @param function The aggregation function to use.
+ * @param rate If true, the rate of the series will be used instead of the
+ * actual values.
+ * @param rate_options If included specifies additional options that are used
+ * when calculating and graph rate values
+ * @throws NoSuchUniqueName if the name of a metric, or a tag name/value
+ * does not exist.
+ * @since 2.4
+ */
+ @Override
+ public void setTimeSeries(String metric,
+ Map tags,
+ Aggregator function,
+ boolean rate,
+ RateOptions rate_options) throws NoSuchUniqueName {
+ if (rollupQuery != null) {
+ rollupQuery.setTimeSeries(metric, tags, function, rate, rate_options);
+ }
+ rawQuery.setTimeSeries(metric, tags, function, rate, rate_options);
+ }
+
+ /**
+ * Sets the time series to the query.
+ *
+ * @param metric The metric to retrieve from the TSDB.
+ * @param tags The set of tags of interest.
+ * @param function The aggregation function to use.
+ * @param rate If true, the rate of the series will be used instead of the
+ * actual values.
+ * @throws NoSuchUniqueName if the name of a metric, or a tag name/value
+ * does not exist.
+ */
+ @Override
+ public void setTimeSeries(String metric, Map tags, Aggregator function, boolean rate) throws NoSuchUniqueName {
+ if (rollupQuery != null) {
+ rollupQuery.setTimeSeries(metric, tags, function, rate);
+ }
+ rawQuery.setTimeSeries(metric, tags, function, rate);
+ }
+
+ /**
+ * Sets up a query for the given timeseries UIDs. For now, all TSUIDs in the
+ * group must share a common metric. This is to avoid issues where the scanner
+ * may have to traverse the entire data table if one TSUID has a metric of
+ * 000001 and another has a metric of FFFFFF. After modifying the query code
+ * to run asynchronously and use different scanners, we can allow different
+ * TSUIDs.
+ * Note: This method will not check to determine if the TSUIDs are
+ * valid, since that wastes time and we *assume* that the user provides TSUIDs
+ * that are up to date.
+ *
+ * @param tsuids A list of one or more TSUIDs to scan for
+ * @param function The aggregation function to use on results
+ * @param rate Whether or not the results should be converted to a rate
+ * @throws IllegalArgumentException if the tsuid list is null, empty or the
+ * TSUIDs do not share a common metric
+ * @since 2.4
+ */
+ @Override
+ public void setTimeSeries(List tsuids, Aggregator function, boolean rate) {
+ if (rollupQuery != null) {
+ rollupQuery.setTimeSeries(tsuids, function, rate);
+ }
+ rawQuery.setTimeSeries(tsuids, function, rate);
+ }
+
+ /**
+ * Sets up a query for the given timeseries UIDs. For now, all TSUIDs in the
+ * group must share a common metric. This is to avoid issues where the scanner
+ * may have to traverse the entire data table if one TSUID has a metric of
+ * 000001 and another has a metric of FFFFFF. After modifying the query code
+ * to run asynchronously and use different scanners, we can allow different
+ * TSUIDs.
+ * Note: This method will not check to determine if the TSUIDs are
+ * valid, since that wastes time and we *assume* that the user provides TSUIDs
+ * that are up to date.
+ *
+ * @param tsuids A list of one or more TSUIDs to scan for
+ * @param function The aggregation function to use on results
+ * @param rate Whether or not the results should be converted to a rate
+ * @param rate_options If included specifies additional options that are used
+ * when calculating and graph rate values
+ * @throws IllegalArgumentException if the tsuid list is null, empty or the
+ * TSUIDs do not share a common metric
+ * @since 2.4
+ */
+ @Override
+ public void setTimeSeries(List tsuids, Aggregator function, boolean rate, RateOptions rate_options) {
+ if (rollupQuery != null) {
+ rollupQuery.setTimeSeries(tsuids, function, rate, rate_options);
+ }
+ rawQuery.setTimeSeries(tsuids, function, rate, rate_options);
+ }
+
+ /**
+ * Prepares a query against HBase by setting up group bys and resolving
+ * strings to UIDs asynchronously. This replaces calls to all of the setters
+ * like the {@link setTimeSeries}, {@link setStartTime}, etc.
+ * Make sure to wait on the deferred return before calling {@link runAsync}.
+ *
+ * @param query The main query to fetch the start and end time from
+ * @param index The index of which sub query we're executing
+ * @return A deferred to wait on for UID resolution. The result doesn't have
+ * any meaning and can be discarded.
+ * @throws IllegalArgumentException if the query was missing sub queries or
+ * the index was out of bounds.
+ * @throws NoSuchUniqueName if the name of a metric, or a tag name/value
+ * does not exist. (Bubbles up through the deferred)
+ * @since 2.4
+ */
+ @Override
+ public Deferred configureFromQuery(TSQuery query, int index) {
+ return configureFromQuery(query, index, false);
+ }
+
+ @Override
+ public Deferred configureFromQuery(TSQuery query, int index, boolean force_raw) {
+ if (force_raw) {
+ throw new UnsupportedOperationException("Not implemented yet");
+ }
+
+ if (!rollupQuery.needsSplitting()) {
+ return rollupResolution;
+ }
+
+ rawQuery = new TsdbQuery(tsdb);
+ rawResolution = rollupQuery.split(query, index, rawQuery);
+
+ if (rollupQuery.getRollupQuery().getLastRollupTimestampSeconds() * 1000L < rollupQuery.getStartTime()) {
+ // We're looking at a query that would normally hit a rollup table, but the table doesn't
+ // have data guaranteed to be available for the requested time period or any part of it
+ // (i.e. the last guaranteed rollup point is before the query actually starts)
+ // So we won't bother running it.
+ rollupQuery = null;
+ }
+
+ return Deferred.group(rollupResolution, rawResolution).addCallback(new GroupCallback());
+ }
+
+ /**
+ * Downsamples the results by specifying a fixed interval between points.
+ *
+ * Technically, downsampling means reducing the sampling interval. Here
+ * the idea is similar. Instead of returning every single data point that
+ * matched the query, we want one data point per fixed time interval. The
+ * way we get this one data point is by aggregating all the data points of
+ * that interval together using an {@link Aggregator}. This enables you
+ * to compute things like the 5-minute average or 10 minute 99th percentile.
+ *
+ * @param interval Number of seconds wanted between each data point.
+ * @param downsampler Aggregation function to use to group data points
+ */
+ @Override
+ public void downsample(long interval, Aggregator downsampler) {
+ if (rollupQuery != null) {
+ rollupQuery.downsample(interval, downsampler);
+ }
+ rawQuery.downsample(interval, downsampler);
+ }
+
+ /**
+ * Sets an optional downsampling function on this query
+ *
+ * @param interval The interval, in milliseconds to rollup data points
+ * @param downsampler An aggregation function to use when rolling up data points
+ * @param fill_policy Policy specifying whether to interpolate or to fill
+ * missing intervals with special values.
+ * @throws NullPointerException if the aggregation function is null
+ * @throws IllegalArgumentException if the interval is not greater than 0
+ * @since 2.4
+ */
+ @Override
+ public void downsample(long interval, Aggregator downsampler, FillPolicy fill_policy) {
+ if (rollupQuery != null) {
+ rollupQuery.downsample(interval, downsampler, fill_policy);
+ }
+ rawQuery.downsample(interval, downsampler, fill_policy);
+ }
+
+ /**
+ * Executes the query asynchronously
+ *
+ * @return The data points matched by this query.
+ *
+ * Each element in the non-{@code null} but possibly empty array returned
+ * corresponds to one time series for which some data points have been
+ * matched by the query.
+ * @throws HBaseException if there was a problem communicating with HBase to
+ * perform the search.
+ * @since 1.2
+ */
+ @Override
+ public Deferred runAsync() throws HBaseException {
+ Deferred rollupResults = Deferred.fromResult(new DataPoints[0]);
+ if (rollupQuery != null) {
+ rollupResults = rollupQuery.runAsync();
+ }
+ Deferred rawResults = rawQuery.runAsync();
+
+ return Deferred.groupInOrder(Arrays.asList(rollupResults, rawResults)).addCallback(new RunCB());
+ }
+
+ /**
+ * Runs this query asynchronously.
+ *
+ * @return The data points matched by this query and applied with percentile calculation
+ *
+ * Each element in the non-{@code null} but possibly empty array returned
+ * corresponds to one time series for which some data points have been
+ * matched by the query.
+ * @throws HBaseException if there was a problem communicating with HBase to
+ * perform the search.
+ * @throws IllegalStateException if the query is not a histogram query
+ */
+ @Override
+ public Deferred runHistogramAsync() throws HBaseException {
+ Deferred rollupResults = Deferred.fromResult(new DataPoints[0]);
+ if (rollupQuery != null) {
+ rollupResults = rollupQuery.runHistogramAsync();
+ } Deferred rawResults = rawQuery.runHistogramAsync();
+
+ return Deferred.groupInOrder(Arrays.asList(rollupResults, rawResults)).addCallback(new RunCB());
+ }
+
+ /**
+ * Returns an index for this sub-query in the original set of queries.
+ *
+ * @return A zero based index.
+ * @since 2.4
+ */
+ @Override
+ public int getQueryIdx() {
+ return rawQuery.getQueryIdx();
+ }
+
+ /**
+ * Check this is a histogram query or not
+ *
+ * @return
+ */
+ @Override
+ public boolean isHistogramQuery() {
+ return rawQuery.isHistogramQuery();
+ }
+
+ /**
+ * Check this is a rollup query or not
+ *
+ * @return Whether or not this is a rollup query
+ * @since 2.4
+ */
+ @Override
+ public boolean isRollupQuery() {
+ return rollupQuery != null && RollupQuery.isValidQuery(rollupQuery.getRollupQuery());
+ }
+
+ /**
+ * @since 2.4
+ */
+ @Override
+ public boolean needsSplitting() {
+ // No further splitting supported
+ return false;
+ }
+
+ /**
+ * Set the percentile calculation parameters for this query if this is
+ * a histogram query
+ *
+ * @param percentiles
+ */
+ @Override
+ public void setPercentiles(List percentiles) {
+ if (rollupQuery != null) {
+ rollupQuery.setPercentiles(percentiles);
+ }
+ rawQuery.setPercentiles(percentiles);
+ }
+
+ private class RunCB implements Callback> {
+
+ private ByteMap makeSpanGroupMap(DataPoints[] dataPointsArray) {
+ ByteMap map = new ByteMap<>();
+
+ for (DataPoints points : dataPointsArray) {
+ if (!(points instanceof SpanGroup)) {
+ throw new IllegalArgumentException("Only SpanGroups implemented");
+ }
+ SpanGroup spanGroup = (SpanGroup) points;
+ map.put(spanGroup.group(), spanGroup);
+ }
+
+ return map;
+ }
+
+ private DataPoints[] merge(DataPoints[] rollup, DataPoints[] raw) {
+ ByteMap rollupResults = makeSpanGroupMap(rollup);
+ ByteMap rawResults = makeSpanGroupMap(raw);
+
+ TreeSet allGroups = new TreeSet<>(Bytes.MEMCMP);
+ allGroups.addAll(rollupResults.keySet());
+ allGroups.addAll(rawResults.keySet());
+
+ List results = new ArrayList<>(allGroups.size());
+
+ for (byte[] group : allGroups) {
+ SpanGroup rawGroup = rawResults.get(group);
+ SpanGroup rollupGroup = rollupResults.get(group);
+ results.add(new SplitRollupSpanGroup(rollupGroup, rawGroup));
+ }
+
+ return results.toArray(new DataPoints[0]);
+ }
+
+ /**
+ * After both queries have run, merge their results
+ *
+ * @param dataPointArrays The results from both queries
+ * @return The merged data points
+ */
+ @Override
+ public DataPoints[] call(ArrayList dataPointArrays) {
+ DataPoints[] rollupResults = dataPointArrays.get(0);
+ DataPoints[] rawResults = dataPointArrays.get(1);
+
+ return merge(rollupResults, rawResults);
+ }
+ }
+}
diff --git a/src/core/SplitRollupSpanGroup.java b/src/core/SplitRollupSpanGroup.java
new file mode 100644
index 0000000000..dd6ab87830
--- /dev/null
+++ b/src/core/SplitRollupSpanGroup.java
@@ -0,0 +1,417 @@
+package net.opentsdb.core;
+
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import net.opentsdb.meta.Annotation;
+import org.hbase.async.Bytes;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SplitRollupSpanGroup extends AbstractSpanGroup {
+ private final List spanGroups = new ArrayList<>();
+
+ public SplitRollupSpanGroup(SpanGroup... groups) {
+ for (SpanGroup group : groups) {
+ if (group != null) {
+ spanGroups.add(group);
+ }
+ }
+
+ if (spanGroups.isEmpty()) {
+ throw new IllegalArgumentException("At least one SpanGroup must be non-null");
+ }
+ }
+
+ /**
+ * Returns the name of the series.
+ */
+ @Override
+ public String metricName() {
+ return spanGroups.get(0).metricName();
+ }
+
+ /**
+ * Returns the name of the series.
+ *
+ * @since 1.2
+ */
+ @Override
+ public Deferred metricNameAsync() {
+ return spanGroups.get(0).metricNameAsync();
+ }
+
+ /**
+ * @return the metric UID
+ * @since 2.3
+ */
+ @Override
+ public byte[] metricUID() {
+ return spanGroups.get(0).metricUID();
+ }
+
+ /**
+ * Returns the tags associated with these data points.
+ *
+ * @return A non-{@code null} map of tag names (keys), tag values (values).
+ */
+ @Override
+ public Map getTags() {
+ Map tags = new HashMap<>();
+
+ for (SpanGroup group : spanGroups) {
+ tags.putAll(group.getTags());
+ }
+
+ return tags;
+ }
+
+ /**
+ * Returns the tags associated with these data points.
+ *
+ * @return A non-{@code null} map of tag names (keys), tag values (values).
+ * @since 1.2
+ */
+ @Override
+ public Deferred> getTagsAsync() {
+ class GetTagsCB implements Callback, ArrayList>> {
+ @Override
+ public Map call(ArrayList> resolvedTags) throws Exception {
+ Map tags = new HashMap<>();
+ for (Map groupTags : resolvedTags) {
+ tags.putAll(groupTags);
+ }
+ return tags;
+ }
+ }
+
+ List>> deferreds = new ArrayList<>(spanGroups.size());
+
+ for (SpanGroup group : spanGroups) {
+ deferreds.add(group.getTagsAsync());
+ }
+
+ return Deferred.groupInOrder(deferreds).addCallback(new GetTagsCB());
+ }
+
+ /**
+ * Returns a map of tag pairs as UIDs.
+ * When used on a span or row, it returns the tag set. When used on a span
+ * group it will return only the tag pairs that are common across all
+ * time series in the group.
+ *
+ * @return A potentially empty map of tagk to tagv pairs as UIDs
+ * @since 2.2
+ */
+ @Override
+ public Bytes.ByteMap getTagUids() {
+ Bytes.ByteMap tagUids = new Bytes.ByteMap<>();
+
+ for (SpanGroup group : spanGroups) {
+ tagUids.putAll(group.getTagUids());
+ }
+
+ return tagUids;
+ }
+
+ /**
+ * Returns the tags associated with some but not all of the data points.
+ *
+ * When this instance represents the aggregation of multiple time series
+ * (same metric but different tags), {@link #getTags} returns the tags that
+ * are common to all data points (intersection set) whereas this method
+ * returns all the tags names that are not common to all data points (union
+ * set minus the intersection set, also called the symmetric difference).
+ *
+ * If this instance does not represent an aggregation of multiple time
+ * series, the list returned is empty.
+ *
+ * @return A non-{@code null} list of tag names.
+ */
+ @Override
+ public List getAggregatedTags() {
+ List aggregatedTags = new ArrayList<>();
+
+ for (SpanGroup group : spanGroups) {
+ aggregatedTags.addAll(group.getAggregatedTags());
+ }
+
+ return aggregatedTags;
+ }
+
+ /**
+ * Returns the tags associated with some but not all of the data points.
+ *
+ * When this instance represents the aggregation of multiple time series
+ * (same metric but different tags), {@link #getTags} returns the tags that
+ * are common to all data points (intersection set) whereas this method
+ * returns all the tags names that are not common to all data points (union
+ * set minus the intersection set, also called the symmetric difference).
+ *
+ * If this instance does not represent an aggregation of multiple time
+ * series, the list returned is empty.
+ *
+ * @return A non-{@code null} list of tag names.
+ * @since 1.2
+ */
+ @Override
+ public Deferred> getAggregatedTagsAsync() {
+ class GetAggregatedTagsCB implements Callback, ArrayList>> {
+ @Override
+ public List call(ArrayList> resolvedTags) throws Exception {
+ List aggregatedTags = new ArrayList<>();
+ for (List groupTags : resolvedTags) {
+ aggregatedTags.addAll(groupTags);
+ }
+ return aggregatedTags;
+ }
+ }
+
+ List>> deferreds = new ArrayList<>(spanGroups.size());
+ for (SpanGroup group : spanGroups) {
+ deferreds.add(group.getAggregatedTagsAsync());
+ }
+
+ return Deferred.groupInOrder(deferreds).addCallback(new GetAggregatedTagsCB());
+ }
+
+ /**
+ * Returns the tagk UIDs associated with some but not all of the data points.
+ *
+ * @return a non-{@code null} list of tagk UIDs.
+ * @since 2.3
+ */
+ @Override
+ public List getAggregatedTagUids() {
+ List aggTagUids = new ArrayList<>();
+
+ for (SpanGroup group : spanGroups) {
+ aggTagUids.addAll(group.getAggregatedTagUids());
+ }
+
+ return aggTagUids;
+ }
+
+ /**
+ * Returns a list of unique TSUIDs contained in the results
+ *
+ * @return an empty list if there were no results, otherwise a list of TSUIDs
+ */
+ @Override
+ public List getTSUIDs() {
+ List tsuids = new ArrayList<>();
+
+ for (SpanGroup group : spanGroups) {
+ tsuids.addAll(group.getTSUIDs());
+ }
+
+ return tsuids;
+ }
+
+ /**
+ * Compiles the annotations for each span into a new array list
+ *
+ * @return Null if none of the spans had any annotations, a list if one or
+ * more were found
+ */
+ @Override
+ public List getAnnotations() {
+ List annotations = new ArrayList<>();
+
+ for (SpanGroup group : spanGroups) {
+ List groupAnnotations = group.getAnnotations();
+ if (groupAnnotations != null) {
+ annotations.addAll(group.getAnnotations());
+ }
+ }
+
+ return annotations;
+ }
+
+ /**
+ * Returns the number of data points.
+ *
+ * This method must be implemented in {@code O(1)} or {@code O(n)}
+ * where n = {@link #aggregatedSize} > 0.
+ *
+ * @return A positive integer.
+ */
+ @Override
+ public int size() {
+ int size = 0;
+ for (SpanGroup group : spanGroups) {
+ size += group.size();
+ }
+ return size;
+ }
+
+ /**
+ * Returns the number of data points aggregated in this instance.
+ *
+ * When this instance represents the aggregation of multiple time series
+ * (same metric but different tags), {@link #size} returns the number of data
+ * points after aggregation, whereas this method returns the number of data
+ * points before aggregation.
+ *
+ * If this instance does not represent an aggregation of multiple time
+ * series, then 0 is returned.
+ *
+ * @return A positive integer.
+ */
+ @Override
+ public int aggregatedSize() {
+ int aggregatedSize = 0;
+ for (SpanGroup group : spanGroups) {
+ aggregatedSize += group.aggregatedSize();
+ }
+ return aggregatedSize;
+ }
+
+ /**
+ * Returns a zero-copy view to go through {@code size()} data points.
+ *
+ * The iterator returned must return each {@link DataPoint} in {@code O(1)}.
+ * The {@link DataPoint} returned must not be stored and gets
+ * invalidated as soon as {@code next} is called on the iterator. If you
+ * want to store individual data points, you need to copy the timestamp
+ * and value out of each {@link DataPoint} into your own data structures.
+ */
+ @Override
+ public SeekableView iterator() {
+ List iterators = new ArrayList<>();
+ for (SpanGroup group : spanGroups) {
+ iterators.add(group.iterator());
+ }
+ return new SeekableViewChain(iterators);
+ }
+
+ /**
+ * Returns the timestamp associated with the {@code i}th data point.
+ * The first data point has index 0.
+ *
+ * This method must be implemented in
+ * O({@link #aggregatedSize}) or better.
+ *
+ * It is guaranteed that timestamp(i) < timestamp(i+1)
+ *
+ * @param i
+ * @return A strictly positive integer.
+ * @throws IndexOutOfBoundsException if {@code i} is not in the range
+ * [0, {@link #size} - 1]
+ */
+ @Override
+ public long timestamp(int i) {
+ return getDataPoint(i).timestamp();
+ }
+
+ /**
+ * Tells whether or not the {@code i}th value is of integer type.
+ * The first data point has index 0.
+ *
+ * This method must be implemented in
+ * O({@link #aggregatedSize}) or better.
+ *
+ * @param i
+ * @return {@code true} if the {@code i}th value is of integer type,
+ * {@code false} if it's of floating point type.
+ * @throws IndexOutOfBoundsException if {@code i} is not in the range
+ * [0, {@link #size} - 1]
+ */
+ @Override
+ public boolean isInteger(int i) {
+ return getDataPoint(i).isInteger();
+ }
+
+ /**
+ * Returns the value of the {@code i}th data point as a long.
+ * The first data point has index 0.
+ *
+ * This method must be implemented in
+ * O({@link #aggregatedSize}) or better.
+ * Use {@link #iterator} to get successive {@code O(1)} accesses.
+ *
+ * @param i
+ * @throws IndexOutOfBoundsException if {@code i} is not in the range
+ * [0, {@link #size} - 1]
+ * @throws ClassCastException if the
+ * {@link #isInteger isInteger(i)} == false.
+ * @see #iterator
+ */
+ @Override
+ public long longValue(int i) {
+ return getDataPoint(i).longValue();
+ }
+
+ /**
+ * Returns the value of the {@code i}th data point as a float.
+ * The first data point has index 0.
+ *
+ * This method must be implemented in
+ * O({@link #aggregatedSize}) or better.
+ * Use {@link #iterator} to get successive {@code O(1)} accesses.
+ *
+ * @param i
+ * @throws IndexOutOfBoundsException if {@code i} is not in the range
+ * [0, {@link #size} - 1]
+ * @throws ClassCastException if the
+ * {@link #isInteger isInteger(i)} == true.
+ * @see #iterator
+ */
+ @Override
+ public double doubleValue(int i) {
+ return getDataPoint(i).doubleValue();
+ }
+
+ /**
+ * Return the query index that maps this datapoints to the original TSSubQuery.
+ *
+ * @return index of the query in the TSQuery class
+ * @throws UnsupportedOperationException if the implementing class can't map
+ * to a sub query.
+ * @since 2.2
+ */
+ @Override
+ public int getQueryIndex() {
+ return spanGroups.get(0).getQueryIndex();
+ }
+
+ /**
+ * Return whether these data points are the result of the percentile calculation
+ * on the histogram data points. The client can call {@code getPercentile} to get
+ * the percentile calculation parameter.
+ *
+ * @return true or false
+ * @since 2.4
+ */
+ @Override
+ public boolean isPercentile() {
+ return spanGroups.get(0).isPercentile();
+ }
+
+ /**
+ * Return the percentile calculation parameter. This interface and {@code isPercentile} are used
+ * to convert {@code HistogramDataPoints} to {@code DataPoints}
+ *
+ * @return the percentile parameter
+ * @since 2.4
+ */
+ @Override
+ public float getPercentile() {
+ return spanGroups.get(0).getPercentile();
+ }
+
+ /**
+ * Returns the group the spans in here belong to.
+ *
+ * Returns null if the NONE aggregator was requested in the query
+ * Returns an empty array if there were no group bys and they're all in the same group
+ * Returns the group otherwise
+ *
+ * @return The group
+ */
+ public byte[] group() {
+ return spanGroups.get(0).group();
+ }
+}
diff --git a/src/core/TSDB.java b/src/core/TSDB.java
index 119c207a24..36853cf1d1 100644
--- a/src/core/TSDB.java
+++ b/src/core/TSDB.java
@@ -23,6 +23,8 @@
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
+import org.hbase.async.RegionLocation;
+import org.hbase.async.HBaseRpc;
import com.google.common.base.Strings;
import com.google.common.io.Files;
@@ -32,6 +34,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.hbase.async.AtomicIncrementRequest;
import org.hbase.async.AppendRequest;
import org.hbase.async.Bytes;
import org.hbase.async.Bytes.ByteMap;
@@ -101,11 +104,40 @@ public final class TSDB {
/** The operation mode (role) of the TSD. */
public enum OperationMode {
- READWRITE,
- READONLY,
- WRITEONLY
+ READWRITE(true, true),
+ READONLY(true, false),
+ WRITEONLY(false, true);
+
+ private final boolean read;
+ private final boolean write;
+
+ OperationMode(boolean read, boolean write) {
+ this.read = read;
+ this.write = write;
+ }
+
+ /** Whether this mode allows reading */
+ public boolean isRead() {
+ return read;
+ }
+
+ /** Whether this mode allows writing */
+ public boolean isWrite() {
+ return write;
+ }
}
-
+
+ /** Whether tables are fully available, partially available, or unavailable.
+ *
+ * The order matters, since we do ordinal comparison—lower should mean
+ * less available.
+ */
+ public enum TableAvailability {
+ NONE,
+ PARTIAL,
+ FULL,
+ }
+
/** Client for the HBase cluster to use. */
final HBaseClient client;
@@ -134,6 +166,9 @@ public enum OperationMode {
/** Timer used for various tasks such as idle timeouts or query timeouts */
private final HashedWheelTimer timer;
+ /** RpcResponder for doing response asynchronously*/
+ private final RpcResponder rpcResponder;
+
/**
* Row keys that need to be compacted.
* Whenever we write a new data point to a row, we add the row key to this
@@ -183,7 +218,13 @@ public enum OperationMode {
/** Whether or not to block writing of derived rollups/pre-ags */
private final boolean rollups_block_derived;
-
+
+ /**
+ * Whether or not to enable splitting rollup queries if the rollup table is lagging
+ * Global config setting: tsd.rollups.split_query.enable = true
+ */
+ private final boolean rollups_split_queries;
+
/** An optional histogram manger used when the TSD will be dealing with
* histograms and sketches. Instantiated ONLY if
* {@link #initializePlugins(boolean)} was called.*/
@@ -312,6 +353,7 @@ public TSDB(final HBaseClient client, final Config config) {
agg_tag_key = config.getString("tsd.rollups.agg_tag_key");
raw_agg_tag_value = config.getString("tsd.rollups.raw_agg_tag_value");
rollups_block_derived = config.getBoolean("tsd.rollups.block_derived");
+ rollups_split_queries = config.getBoolean("tsd.rollups.split_query.enable");
} else {
rollup_config = null;
default_interval = null;
@@ -319,6 +361,7 @@ public TSDB(final HBaseClient client, final Config config) {
agg_tag_key = null;
raw_agg_tag_value = null;
rollups_block_derived = false;
+ rollups_split_queries = false;
}
QueryStats.setEnableDuplicates(
@@ -343,7 +386,10 @@ public TSDB(final HBaseClient client, final Config config) {
// set any extra tags from the config for stats
StatsCollector.setGlobalTags(config);
-
+
+
+ rpcResponder = new RpcResponder(config);
+
LOG.debug(config.dumpConfiguration());
}
@@ -549,7 +595,7 @@ public void initializePlugins(final boolean init_rpcs) {
uid_filter.getClass().getCanonicalName() + "] version: "
+ uid_filter.version());
}
-
+
// finally load the histo manager after plugins have been loaded.
if (config.hasProperty("tsd.core.histograms.config")) {
histogram_manager = new HistogramCodecManager(this);
@@ -566,7 +612,7 @@ public void initializePlugins(final boolean init_rpcs) {
public final Authentication getAuth() {
return this.authentication;
}
-
+
/**
* Returns the configured HBase client
* @return The HBase client
@@ -728,6 +774,145 @@ public Deferred> checkNecessaryTablesExist() {
return Deferred.group(checks);
}
+ /* Suffix for queries from availability check below. */
+ private static byte[] PROBE_SUFFIX = {
+ ':', 'A', 's', 'y', 'n', 'c', 'H', 'B', 'a', 's', 'e',
+ '~', 'p', 'r', 'o', 'b', 'e', '~', '<', ';', '_', '<',
+ };
+
+ /**
+ * Get availability status of regions for a table.
+ *
+ * Implemented as separate method so we can override it in unit tests.
+ *
+ * @return Per-region availability.
+ *
+ * @since 2.5
+ */
+ Deferred> getTableRegionAvailability(String table) {
+ final String table_id = config.getString(table);
+
+ /** Convert result to true. */
+ final class SuccessToBoolCallback implements Callback> {
+ @Override
+ public Boolean call(final ArrayList o) {
+ LOG.info("Check HBase availability, got success.");
+ return true;
+ }
+ }
+
+ /** Convert error result to false. */
+ final class FailureToBoolCallback implements Callback {
+ @Override
+ public Boolean call(final Exception e) {
+ LOG.error("Check HBase availability, got error:", e);
+ return false;
+ }
+ }
+
+ final SuccessToBoolCallback successCB = new SuccessToBoolCallback();
+ final FailureToBoolCallback failureCB = new FailureToBoolCallback();
+
+ /** Lookup availability of each region. */
+ final class RegionInfoCallback implements Callback>,List> {
+ @Override
+ public Deferred> call(final List regions) {
+ LOG.info("Availability check got this many regions: " + regions.size());
+ ArrayList> available = new ArrayList>();
+ for (RegionLocation region : regions) {
+ // Use suffix so we don't hit real data:
+ final byte[] key = region.startKey();
+ final byte[] testKey = new byte[key.length + 64];
+ System.arraycopy(key, 0, testKey, 0, key.length);
+ System.arraycopy(PROBE_SUFFIX, 0,
+ testKey, testKey.length - PROBE_SUFFIX.length,
+ PROBE_SUFFIX.length);
+ LOG.debug("Checking region with start key " + testKey + " end key " + region.stopKey());
+ GetRequest probe = new GetRequest(table_id, testKey);
+ // If we don't get a response within 1 second, assume the region is
+ // unavailable.
+ probe.setTimeout(1000);
+ probe.setFailfast(true);
+ available.add(client.get(probe).addCallbacks(successCB, failureCB));
+ }
+ return Deferred.group(available);
+ }
+ }
+
+ return client.locateRegions(config.getString(table))
+ .addCallbackDeferring(new RegionInfoCallback());
+ }
+
+ /**
+ * Check for full or partial data and UID tables availability in HBase.
+ *
+ * @return Status of table availability.
+ *
+ * @since 2.5
+ */
+ public Deferred checkNecessaryTablesAvailability() {
+ /** Convert list of booleans (indicating a region being available) into full
+ * (all were available), partial (some were available), none (none were
+ * available).
+ */
+ final class TableAvailabilityCB implements Callback> {
+ @Override
+ public TableAvailability call(final ArrayList available) {
+ if (available.size() == 0) {
+ return TableAvailability.NONE;
+ }
+ boolean hasAvailable = false;
+ boolean hasUnavailable = false;
+ for (Boolean regionAvailable : available) {
+ if (regionAvailable) {
+ hasAvailable = true;
+ } else {
+ hasUnavailable = true;
+ }
+ }
+ if (hasAvailable && hasUnavailable) {
+ return TableAvailability.PARTIAL;
+ } else if (hasAvailable) {
+ return TableAvailability.FULL;
+ } else {
+ return TableAvailability.NONE;
+ }
+ }
+ }
+
+ /** If getting regions fails, availability is NONE. */
+ final class FailedRegionInfoCallback implements Callback {
+ @Override
+ public TableAvailability call(final Exception e) {
+ LOG.error("Failed to get regions during table availability check", e);
+ return TableAvailability.NONE;
+ }
+ }
+
+ ArrayList> tables = new ArrayList>();
+ tables.add(getTableRegionAvailability("tsd.storage.hbase.uid_table")
+ .addCallbacks(new TableAvailabilityCB(), new FailedRegionInfoCallback()));
+ tables.add(getTableRegionAvailability("tsd.storage.hbase.data_table")
+ .addCallbacks(new TableAvailabilityCB(), new FailedRegionInfoCallback()));
+
+ /** Combine availability for two tables by picking the lower of the two. */
+ final class CombineAvailabilityCB implements Callback> {
+ @Override
+ public TableAvailability call(final ArrayList availabilities) {
+ assert availabilities.size() == 2;
+ TableAvailability result = TableAvailability.FULL;
+ for (TableAvailability availability: availabilities) {
+ if (availability.ordinal() < result.ordinal()) {
+ result = availability;
+ }
+ }
+ return result;
+ }
+ }
+
+ return Deferred.group(tables).addCallback(new CombineAvailabilityCB());
+ }
+
/** Number of cache hits during lookups involving UIDs. */
public long uidCacheHits() {
return (metrics.cacheHits() + tag_names.cacheHits()
@@ -808,6 +993,13 @@ public void collectStats(final StatsCollector collector) {
collector.clearExtraTag("class");
}
+ collector.addExtraTag("class", "IncomingDataPoints");
+ try {
+ collector.record("uid.autometric.rejections", IncomingDataPoints.auto_metric_rejection_count, "method=put");
+ } finally {
+ collector.clearExtraTag("class");
+ }
+
collector.addExtraTag("class", "TSDB");
try {
collector.record("datapoints.added", datapoints_added, "type=all");
@@ -906,6 +1098,14 @@ public void collectStats(final StatsCollector collector) {
collector.clearExtraTag("plugin");
}
}
+ if (meta_cache != null) {
+ try {
+ collector.addExtraTag("plugin", "metaCache");
+ meta_cache.collectStats(collector);
+ } finally {
+ collector.clearExtraTag("plugin");
+ }
+ }
}
/** Returns a latency histogram for Put RPCs used to store data points. */
@@ -1006,6 +1206,7 @@ public Deferred addPoint(final String metric,
final long value,
final Map tags) {
final byte[] v;
+
if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) {
v = new byte[] { (byte) value };
} else if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) {
@@ -1194,8 +1395,14 @@ public Deferred call(final Boolean allowed) throws Exception {
result = client.append(point);
} else if (!isHistogram(qualifier)) {
scheduleForCompaction(row, (int) base_time);
- final PutRequest point = RequestBuilder.buildPutRequest(config, table, row, FAMILY, qualifier, value, timestamp);
+ boolean isLong = ((flags & Const.FLAG_FLOAT) == 0x0);
+ if (isLong && config.use_hbase_counters()) {
+ AtomicIncrementRequest counterRequest = new AtomicIncrementRequest(table, row, FAMILY, qualifier, Bytes.getLong(value));
+ result = client.atomicIncrement(counterRequest);
+ } else {
+ final PutRequest point = new PutRequest(table, row, FAMILY, qualifier, value);
result = client.put(point);
+ }
} else {
scheduleForCompaction(row, (int) base_time);
final PutRequest histo_point = new PutRequest(table, row, FAMILY, qualifier, value);
@@ -1657,20 +1864,43 @@ public String toString() {
}
}
+ final class RpcResponsderShutdown implements Callback {
+ @Override
+ public Object call(Object arg) throws Exception {
+ try {
+ TSDB.this.rpcResponder.close();
+ } catch (Exception e) {
+ LOG.error(
+ "Run into unknown exception while closing RpcResponder.", e);
+ } finally {
+ return arg;
+ }
+ }
+ }
+
final class HClientShutdown implements Callback, ArrayList> {
- public Deferred call(final ArrayList args) {
+ public Deferred call(final ArrayList args) {
+ Callback nextCallback;
if (storage_exception_handler != null) {
- return client.shutdown().addBoth(new SEHShutdown());
+ nextCallback = new SEHShutdown();
+ } else {
+ nextCallback = new FinalShutdown();
}
- return client.shutdown().addBoth(new FinalShutdown());
+
+ if (TSDB.this.rpcResponder.isAsync()) {
+ client.shutdown().addBoth(new RpcResponsderShutdown());
+ }
+
+ return client.shutdown().addBoth(nextCallback);
}
- public String toString() {
+
+ public String toString() {
return "shutdown HBase client";
}
}
final class ShutdownErrback implements Callback {
- public Object call(final Exception e) {
+ public Object call(final Exception e) {
final Logger LOG = LoggerFactory.getLogger(ShutdownErrback.class);
if (e instanceof DeferredGroupException) {
final DeferredGroupException ge = (DeferredGroupException) e;
@@ -1684,13 +1914,14 @@ public Object call(final Exception e) {
}
return new HClientShutdown().call(null);
}
- public String toString() {
+
+ public String toString() {
return "shutdown HBase client after error";
}
}
final class CompactCB implements Callback> {
- public Object call(ArrayList compactions) throws Exception {
+ public Object call(ArrayList compactions) throws Exception {
return null;
}
}
@@ -2116,7 +2347,18 @@ public String getRawTagValue() {
return raw_agg_tag_value;
}
- /** @return The optional histogram manager registered to this TSD.
+ /**
+ * Returns whether the global config setting allows splitting rollups queries
+ * if the rollups table to be hit is lagging.
+ *
+ * @return Whether or not splitting rollup queries is enabled
+ * @since 2.4
+ */
+ public boolean isRollupsSplittingEnabled() {
+ return rollups_split_queries;
+ }
+
+ /** @return The optional histogram manager registered to this TSD.
* @since 2.4 */
public HistogramCodecManager histogramManager() {
return histogram_manager;
@@ -2189,4 +2431,87 @@ final Deferred delete(final byte[] key, final byte[][] qualifiers) {
return client.delete(new DeleteRequest(table, key, FAMILY, qualifiers));
}
+ /** Do response by RpcResponder */
+ public void response(Runnable run) {
+ rpcResponder.response(run);
+ }
+
+
+ /**
+ * store a single long value data point in the TSDB as HBase counter.
+ * @param metric A non-empty string.
+ * @param timestamp The timestamp associated with the value.
+ * @param value The value of the data point.
+ * @param tags The tags on this series. This map must be non-empty.
+ * @return A deferred object that indicates the completion of the request.
+ * The {@link Object} has not special meaning and can be {@code null} (think
+ * of it as {@code Deferred}). But you probably want to attach at
+ * least an errback to this {@code Deferred} to handle failures.
+ * @throws IllegalArgumentException if the timestamp is less than or equal
+ * to the previous timestamp added or 0 for the first timestamp, or if the
+ * difference with the previous timestamp is too large.
+ * @throws IllegalArgumentException if the metric name is empty or contains
+ * illegal characters.
+ * @throws IllegalArgumentException if the value is NaN or infinite.
+ * @throws IllegalArgumentException if the tags list is empty or one of the
+ * elements contains illegal characters.
+ * @throws HBaseException (deferred) if there was a problem while persisting
+ * data.
+ */
+ public Deferred addCounter(String metric, long timestamp, long valueAsLong, HashMap tags) {
+
+ final byte[] value = Bytes.fromLong(valueAsLong);
+ final short flags = (short) (value.length - 1);
+
+ // we only accept positive unix epoch timestamps in seconds or milliseconds
+ if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 &&
+ timestamp > 9999999999999L)) {
+ throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad")
+ + " timestamp=" + timestamp
+ + " when trying to add value=" + Arrays.toString(value) + '/' + flags
+ + " to metric=" + metric + ", tags=" + tags);
+ }
+ IncomingDataPoints.checkMetricAndTags(metric, tags);
+ final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags);
+ final long base_time;
+ final byte[] qualifier = Internal.buildQualifier(timestamp, flags);
+
+ if ((timestamp & Const.SECOND_MASK) != 0) {
+ // drop the ms timestamp to seconds to calculate the base timestamp
+ base_time = ((timestamp / 1000) -
+ ((timestamp / 1000) % Const.MAX_TIMESPAN));
+ } else {
+ base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
+ }
+
+ Bytes.setInt(row, (int) base_time, metrics.width());
+ scheduleForCompaction(row, (int) base_time);
+
+ AtomicIncrementRequest counterRequest = new AtomicIncrementRequest(table, row, FAMILY, qualifier, Bytes.getLong(value));
+ client.atomicIncrement(counterRequest);
+
+ if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() &&
+ !config.enable_tsuid_tracking() && rt_publisher == null) {
+ return null;
+ }
+
+ final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH,
+ Const.TIMESTAMP_BYTES);
+
+ // for busy TSDs we may only enable TSUID tracking, storing a 1 in the
+ // counter field for a TSUID with the proper timestamp. If the user would
+ // rather have TSUID incrementing enabled, that will trump the PUT
+ if (config.enable_tsuid_tracking() && !config.enable_tsuid_incrementing()) {
+ final PutRequest tracking = new PutRequest(meta_table, tsuid,
+ TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1));
+ client.put(tracking);
+ } else if (config.enable_tsuid_incrementing() || config.enable_realtime_ts()) {
+ TSMeta.incrementAndGetCounter(TSDB.this, tsuid);
+ }
+
+ if (rt_publisher != null) {
+ rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags);
+ }
+ return null;
+ }
}
diff --git a/src/core/TSQuery.java b/src/core/TSQuery.java
index e571004d66..9e61ffeedf 100644
--- a/src/core/TSQuery.java
+++ b/src/core/TSQuery.java
@@ -176,9 +176,9 @@ public void validateAndSetQuery() {
} else {
end_time = System.currentTimeMillis();
}
- if (end_time <= start_time) {
+ if (end_time < start_time) {
throw new IllegalArgumentException(
- "End time [" + end_time + "] must be greater than the start time ["
+ "End time [" + end_time + "] must be greater than or equal to the start time ["
+ start_time +"]");
}
@@ -240,8 +240,15 @@ public Deferred buildQueriesAsync(final TSDB tsdb) {
final List> deferreds =
new ArrayList>(queries.size());
for (int i = 0; i < queries.size(); i++) {
- final Query query = tsdb.newQuery();
- deferreds.add(query.configureFromQuery(this, i));
+ Query query = tsdb.newQuery();
+ Deferred resolution = query.configureFromQuery(this, i);
+
+ if (query.needsSplitting() && (query instanceof TsdbQuery)) {
+ query = new SplitRollupQuery(tsdb, (TsdbQuery) query, resolution);
+ resolution = query.configureFromQuery(this, i);
+ }
+ deferreds.add(resolution);
+
tsdb_queries[i] = query;
}
diff --git a/src/core/TsdbQuery.java b/src/core/TsdbQuery.java
index a0dd841d6a..4c8d60c69f 100644
--- a/src/core/TsdbQuery.java
+++ b/src/core/TsdbQuery.java
@@ -20,6 +20,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.SortedMap;
import java.util.TreeMap;
import org.slf4j.Logger;
@@ -60,7 +61,7 @@
/**
* Non-synchronized implementation of {@link Query}.
*/
-final class TsdbQuery implements Query {
+final class TsdbQuery extends AbstractQuery {
private static final Logger LOG = LoggerFactory.getLogger(TsdbQuery.class);
@@ -97,22 +98,22 @@ final class TsdbQuery implements Query {
/** End time (UNIX timestamp in seconds) on 32 bits ("unsigned" int). */
private long end_time = UNSET;
-
+
/** Whether or not to delete the queried data */
private boolean delete;
/** ID of the metric being looked up. */
private byte[] metric;
-
+
/** Row key regex to pass to HBase if we have tags or TSUIDs */
private String regex;
-
+
/** Whether or not to enable the fuzzy row filter for Hbase */
private boolean enable_fuzzy_filter;
-
+
/** Whether or not the user wants to use the fuzzy filter */
private boolean override_fuzzy_filter;
-
+
/**
* Tags by which we must group the results.
* Each element is a tag ID.
@@ -131,7 +132,7 @@ final class TsdbQuery implements Query {
/** Specifies the various options for rate calculations */
private RateOptions rate_options;
-
+
/** Aggregator function to use. */
private Aggregator aggregator;
@@ -140,55 +141,55 @@ final class TsdbQuery implements Query {
/** Rollup interval and aggregator, null if not applicable. */
private RollupQuery rollup_query;
-
+
/** Map of RollupInterval objects in the order of next best match
* like 1d, 1h, 10m, 1m, for rollup of 1d. */
private List best_match_rollups;
-
+
/** How to use the rollup data */
private ROLLUP_USAGE rollup_usage = ROLLUP_USAGE.ROLLUP_NOFALLBACK;
-
- /** Search the query on pre-aggregated table directly instead of post fetch
+
+ /** Search the query on pre-aggregated table directly instead of post fetch
* aggregation. */
private boolean pre_aggregate;
-
+
/** Optional list of TSUIDs to fetch and aggregate instead of a metric */
private List tsuids;
-
+
/** An index that links this query to the original sub query */
private int query_index;
-
+
/** Tag value filters to apply post scan */
private List filters;
-
+
/** An object for storing stats in regarding the query. May be null */
private QueryStats query_stats;
-
+
/** Whether or not to match series with ONLY the given tags */
private boolean explicit_tags;
-
+
private List percentiles;
-
+
private boolean show_histogram_buckets;
-
+
/** Set at filter resolution time to determine if we can use multi-gets */
private boolean use_multi_gets;
/** Set by the user if they want to bypass multi-gets */
private boolean override_multi_get;
-
+
/** Whether or not to use the search plugin for multi-get resolution. */
private boolean multiget_with_search;
-
+
/** Whether or not to fall back on query failure. */
private boolean search_query_failure;
-
+
/** The maximum number of bytes allowed per query. */
private long max_bytes = 0;
-
+
/** The maximum number of data points allowed per query. */
private long max_data_points = 0;
-
+
/**
* Enum for rollup fallback control.
* @since 2.4
@@ -198,7 +199,7 @@ public static enum ROLLUP_USAGE {
ROLLUP_NOFALLBACK, //Use rollup data, and don't fallback on no data
ROLLUP_FALLBACK, //Use rollup data and fallback to next best match on data
ROLLUP_FALLBACK_RAW; //Use rollup data and fallback to raw on no data
-
+
/**
* Parse and transform a string to ROLLUP_USAGE object
* @param str String to be parsed
@@ -206,7 +207,7 @@ public static enum ROLLUP_USAGE {
*/
public static ROLLUP_USAGE parse(String str) {
ROLLUP_USAGE def = ROLLUP_NOFALLBACK;
-
+
if (str != null) {
try {
def = ROLLUP_USAGE.valueOf(str.toUpperCase());
@@ -216,10 +217,10 @@ public static ROLLUP_USAGE parse(String str) {
+ "uses raw data but don't fallback on no data");
}
}
-
+
return def;
}
-
+
/**
* Whether to fallback to next best match or raw
* @return true means fall back else false
@@ -228,7 +229,7 @@ public boolean fallback() {
return this == ROLLUP_FALLBACK || this == ROLLUP_FALLBACK_RAW;
}
}
-
+
/** Constructor. */
public TsdbQuery(final TSDB tsdb) {
this.tsdb = tsdb;
@@ -248,15 +249,15 @@ public String getRollupTable() {
return "raw";
}
}
-
- /** Search the query on pre-aggregated table directly instead of post fetch
- * aggregation.
- * @since 2.4
+
+ /** Search the query on pre-aggregated table directly instead of post fetch
+ * aggregation.
+ * @since 2.4
*/
public boolean isPreAggregate() {
return this.pre_aggregate;
}
-
+
/**
* Sets the start time for the query
* @param timestamp Unix epoch timestamp in seconds or milliseconds
@@ -265,12 +266,12 @@ public boolean isPreAggregate() {
*/
@Override
public void setStartTime(final long timestamp) {
- if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 &&
+ if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 &&
timestamp > 9999999999999L)) {
throw new IllegalArgumentException("Invalid timestamp: " + timestamp);
- } else if (end_time != UNSET && timestamp >= getEndTime()) {
+ } else if (end_time != UNSET && timestamp > getEndTime()) {
throw new IllegalArgumentException("new start time (" + timestamp
- + ") is greater than or equal to end time: " + getEndTime());
+ + ") is greater than end time: " + getEndTime());
}
start_time = timestamp;
}
@@ -299,9 +300,9 @@ public void setEndTime(final long timestamp) {
if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 &&
timestamp > 9999999999999L)) {
throw new IllegalArgumentException("Invalid timestamp: " + timestamp);
- } else if (start_time != UNSET && timestamp <= getStartTime()) {
+ } else if (start_time != UNSET && timestamp < getStartTime()) {
throw new IllegalArgumentException("new end time (" + timestamp
- + ") is less than or equal to start time: " + getStartTime());
+ + ") is less than start time: " + getStartTime());
}
end_time = timestamp;
}
@@ -428,10 +429,72 @@ public void setTimeSeries(final List tsuids,
public void setExplicitTags(final boolean explicit_tags) {
this.explicit_tags = explicit_tags;
}
-
+
+ /**
+ * Splits this query into one query for the part that is covered by the rollup
+ * table (as defined in its SLA) and one to get the data for the remaining time
+ * range from the raw table.
+ * @param query The original TSQuery as parsed
+ * @param index The index of the TSQuery
+ * @param rawQuery A new TsdbQuery instance that will be configured to hit the raw table
+ * for the correct time range
+ * @return the deferred analogous to {@link TsdbQuery#configureFromQuery(TSQuery, int)}
+ * @throws IllegalStateException if the query is not eligible or splitting is disabled
+ */
+ public Deferred split(final TSQuery query, final int index, final TsdbQuery rawQuery) {
+ if (!needsSplitting()) {
+ throw new IllegalStateException("Query is not eligible for splitting" + this.toString());
+ }
+
+ Deferred rawResolutionDeferred = rawQuery.configureFromQuery(query, index, true);
+
+ long lastRollupTimestampMillis = rollup_query.getLastRollupTimestampSeconds() * 1000L;
+
+ boolean needsRawAndRollupData = QueryUtil.isTimestampAfter(lastRollupTimestampMillis, getStartTime());
+ if (needsRawAndRollupData) {
+ updateRollupSplitTimes(rawQuery, lastRollupTimestampMillis);
+ }
+
+ return rawResolutionDeferred;
+ }
+
+ /**
+ * Updates the timestamp of this query and the corresponding raw part in the case of a split.
+ *
+ * Sets the start and end times for this query so that it hits the rollup table until the given timestamp.
+ * Also updates the passed {@param rawQuery} with the new start time so that it hits the raw table for points from the
+ * given timestamp onwards.
+ *
+ * Makes sure that all timestamps are in milliseconds.
+ *
+ * @param rawQuery The raw query part
+ * @param splitTimestamp The timestamp until when rollup data is guaranteed to be available
+ */
+ private void updateRollupSplitTimes(final TsdbQuery rawQuery, long splitTimestamp) {
+ setEndTime(splitTimestamp);
+
+ boolean isStartTimeInSeconds = (getStartTime() & Const.SECOND_MASK) == 0;
+ if (isStartTimeInSeconds) {
+ setStartTime(getStartTime() * 1000L);
+ }
+
+ boolean isRawEndTimeInSeconds = (rawQuery.getEndTime() & Const.SECOND_MASK) == 0;
+ if (isRawEndTimeInSeconds) {
+ rawQuery.setEndTime(rawQuery.getEndTime() * 1000L);
+ }
+
+ rawQuery.setStartTime(splitTimestamp);
+ }
+
@Override
- public Deferred configureFromQuery(final TSQuery query,
- final int index) {
+ public Deferred configureFromQuery(final TSQuery query,
+ final int index) {
+ return configureFromQuery(query, index, false);
+ }
+
+
+ public Deferred configureFromQuery(final TSQuery query,
+ final int index, boolean force_raw) {
if (query.getQueries() == null || query.getQueries().isEmpty()) {
throw new IllegalArgumentException("Missing sub queries");
}
@@ -476,7 +539,7 @@ public Deferred configureFromQuery(final TSQuery query,
percentiles = sub_query.getPercentiles();
show_histogram_buckets = sub_query.getShowHistogramBuckets();
- if (rollup_usage != ROLLUP_USAGE.ROLLUP_RAW) {
+ if (!force_raw && rollup_usage != ROLLUP_USAGE.ROLLUP_RAW) {
//Check whether the down sampler is set and rollup is enabled
transformDownSamplerToRollupQuery(aggregator, sub_query.getDownsample());
}
@@ -575,7 +638,7 @@ private List> resolveTagFilters() {
return deferreds;
}
}
-
+
// fire off the callback chain by resolving the metric first
return tsdb.metrics.getIdAsync(sub_query.getMetric())
.addCallbackDeferring(new MetricCB());
@@ -586,7 +649,7 @@ private List> resolveTagFilters() {
public void downsample(final long interval, final Aggregator downsampler,
final FillPolicy fill_policy) {
this.downsampler = new DownsamplingSpecification(
- interval, downsampler,fill_policy);
+ interval, downsampler, fill_policy);
}
/**
@@ -703,43 +766,11 @@ private void findGroupBys() {
}
}
}
- /**
- * Executes the query.
- * NOTE: Do not run the same query multiple times. Construct a new query with
- * the same parameters again if needed
- * TODO(cl) There are some strange occurrences when unit testing where the end
- * time, if not set, can change between calls to run()
- * @return An array of data points with one time series per array value
- */
- @Override
- public DataPoints[] run() throws HBaseException {
- try {
- return runAsync().joinUninterruptibly();
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException("Should never be here", e);
- }
- }
-
- @Override
- public DataPoints[] runHistogram() throws HBaseException {
- if (!isHistogramQuery()) {
- throw new RuntimeException("Should never be here");
- }
-
- try {
- return runHistogramAsync().joinUninterruptibly();
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException("Should never be here", e);
- }
- }
-
+
@Override
public Deferred runAsync() throws HBaseException {
Deferred result = null;
+
if (use_multi_gets && override_multi_get) {
result = this.findSpansWithMultiGetter().addCallback(new GroupByAndAggregateCB());
} else {
@@ -776,10 +807,45 @@ public boolean isHistogramQuery() {
if ((this.percentiles != null && this.percentiles.size() > 0) || show_histogram_buckets) {
return true;
}
-
+
return false;
}
-
+
+ @Override
+ public boolean isRollupQuery() {
+ return RollupQuery.isValidQuery(rollup_query);
+ }
+
+ /**
+ * Returns whether this query needs to be split. It does if
+ * - splitting of queries is enabled globally AND
+ * - it can be split (i.e. it's a valid rollups query) AND
+ * - the table it is hitting has an SLA configured that describes the blackout period AND
+ * - the query is actually looking at data from the time beyond the SLA
+ *
+ * @return whether this query needs to be split.
+ * @since 2.4
+ */
+ @Override
+ public boolean needsSplitting() {
+ if (!tsdb.isRollupsSplittingEnabled()) {
+ // Don't split if the global config doesn't allow it
+ return false;
+ }
+
+ if (!isRollupQuery()) {
+ // Don't split if it's hitting the raw table anyway
+ return false;
+ }
+
+ if (rollup_query.getRollupInterval().getMaximumLag() <= 0) {
+ // Don't split if the table doesn't have a maximum lag configured
+ return false;
+ }
+
+ return rollup_query.isInBlackoutPeriod(getEndTime());
+ }
+
/**
* Finds all the {@link Span}s that match this query.
* This is what actually scans the HBase table and loads the data into
@@ -791,7 +857,7 @@ public boolean isHistogramQuery() {
* perform the search.
* @throws IllegalArgumentException if bad data was retrieved from HBase.
*/
- private Deferred> findSpans() throws HBaseException {
+ private Deferred> findSpans() throws HBaseException {
final short metric_width = tsdb.metrics.width();
final TreeMap spans = // The key is a row key from HBase.
new TreeMap(new SpanCmp(
@@ -831,14 +897,14 @@ private Deferred> findSpans() throws HBaseException {
}
}
- private Deferred> findSpansWithMultiGetter() throws HBaseException {
+ private Deferred> findSpansWithMultiGetter() throws HBaseException {
final short metric_width = tsdb.metrics.width();
final TreeMap spans = // The key is a row key from HBase.
new TreeMap(new SpanCmp(metric_width));
scan_start_time = System.nanoTime();
-
- return new MultiGetQuery(tsdb, this, metric, row_key_literals_list,
+
+ return new MultiGetQuery(tsdb, this, metric, row_key_literals_list,
getScanStartTimeSeconds(), getScanEndTimeSeconds(),
tableToBeScanned(), spans, null, 0, rollup_query, query_stats, query_index, 0,
false, search_query_failure).fetch();
@@ -857,7 +923,7 @@ private Deferred> findSpansWithMultiGetter() throws HBaseE
* perform the search.
* @throws IllegalArgumentException if bad data was retreived from HBase.
*/
- private Deferred> findHistogramSpans() throws HBaseException {
+ private Deferred> findHistogramSpans() throws HBaseException {
final short metric_width = tsdb.metrics.width();
final TreeMap histSpans = new TreeMap(new SpanCmp(metric_width));
@@ -896,7 +962,7 @@ private Deferred> findHistogramSpans() throws HBa
}
}
- private Deferred> findHistogramSpansWithMultiGetter() throws HBaseException {
+ private Deferred> findHistogramSpansWithMultiGetter() throws HBaseException {
final short metric_width = tsdb.metrics.width();
// The key is a row key from HBase
final TreeMap histSpans = new TreeMap(new SpanCmp(metric_width));
@@ -913,7 +979,7 @@ private Deferred> findHistogramSpansWithMultiGett
* {@link TsdbQuery#findSpans} to group and sort the results.
*/
private class GroupByAndAggregateCB implements
- Callback>{
+ Callback>{
/**
* Creates the {@link SpanGroup}s to form the final results of this query.
@@ -923,7 +989,7 @@ private class GroupByAndAggregateCB implements
* any 'GROUP BY' formulated in this query.
*/
@Override
- public DataPoints[] call(final TreeMap spans) throws Exception {
+ public DataPoints[] call(final SortedMap spans) throws Exception {
if (query_stats != null) {
query_stats.addStat(query_index, QueryStat.QUERY_SCAN_TIME,
(System.nanoTime() - TsdbQuery.this.scan_start_time));
@@ -953,7 +1019,8 @@ public DataPoints[] call(final TreeMap spans) throws Exception {
getStartTime(),
getEndTime(),
query_index,
- rollup_query);
+ rollup_query,
+ null);
group.add(span);
groups[i++] = group;
}
@@ -973,7 +1040,8 @@ public DataPoints[] call(final TreeMap spans) throws Exception {
getStartTime(),
getEndTime(),
query_index,
- rollup_query);
+ rollup_query,
+ new byte[0]);
if (query_stats != null) {
query_stats.addStat(query_index, QueryStat.GROUP_BY_TIME, 0);
}
@@ -1017,6 +1085,11 @@ public DataPoints[] call(final TreeMap spans) throws Exception {
//LOG.info("Span belongs to group " + Arrays.toString(group) + ": " + Arrays.toString(row));
SpanGroup thegroup = groups.get(group);
if (thegroup == null) {
+ // Copy the array because we're going to keep `group' and overwrite
+ // its contents. So we want the collection to have an immutable copy.
+ final byte[] group_copy = new byte[group.length];
+ System.arraycopy(group, 0, group_copy, 0, group.length);
+
thegroup = new SpanGroup(tsdb, getScanStartTimeSeconds(),
getScanEndTimeSeconds(),
null, rate, rate_options, aggregator,
@@ -1024,11 +1097,8 @@ public DataPoints[] call(final TreeMap spans) throws Exception {
getStartTime(),
getEndTime(),
query_index,
- rollup_query);
- // Copy the array because we're going to keep `group' and overwrite
- // its contents. So we want the collection to have an immutable copy.
- final byte[] group_copy = new byte[group.length];
- System.arraycopy(group, 0, group_copy, 0, group.length);
+ rollup_query,
+ group_copy);
groups.put(group_copy, thegroup);
}
thegroup.add(entry.getValue());
@@ -1048,7 +1118,7 @@ public DataPoints[] call(final TreeMap spans) throws Exception {
* {@link TsdbQuery#findHistogramSpans} to group and sort the results.
*/
private class HistogramGroupByAndAggregateCB implements
- Callback>{
+ Callback>{
/**
* Creates the {@link HistogramSpanGroup}s to form the final results of this query.
@@ -1057,7 +1127,7 @@ private class HistogramGroupByAndAggregateCB implements
* @return A possibly empty array of {@link HistogramSpanGroup}s built according to
* any 'GROUP BY' formulated in this query.
*/
- public DataPoints[] call(final TreeMap spans) throws Exception {
+ public DataPoints[] call(final SortedMap spans) throws Exception {
if (query_stats != null) {
query_stats.addStat(query_index, QueryStat.QUERY_SCAN_TIME,
(System.nanoTime() - TsdbQuery.this.scan_start_time));
@@ -1394,8 +1464,8 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException {
final Scanner scanner = QueryUtil.getMetricScanner(tsdb, salt_bucket, metric,
(int) getScanStartTimeSeconds(), end_time == UNSET
? -1 // Will scan until the end (0xFFF...).
- : (int) getScanEndTimeSeconds(),
- tableToBeScanned(),
+ : (int) getScanEndTimeSeconds(),
+ tableToBeScanned(),
TSDB.FAMILY());
if(tsdb.getConfig().use_otsdb_timestamp()) {
long stTime = (getScanStartTimeSeconds() * 1000);
@@ -1428,7 +1498,7 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException {
new BinaryPrefixComparator(rollup_query.getRollupAgg().toString()
.getBytes(Const.ASCII_CHARSET))));
rollup_filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL,
- new BinaryPrefixComparator(new byte[] {
+ new BinaryPrefixComparator(new byte[] {
(byte) tsdb.getRollupConfig().getIdForAggregator(
rollup_query.getRollupAgg().toString())
})));
@@ -1440,7 +1510,7 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException {
new BinaryPrefixComparator(rollup_query.getRollupAgg().toString()
.getBytes(Const.ASCII_CHARSET))));
filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL,
- new BinaryPrefixComparator(new byte[] {
+ new BinaryPrefixComparator(new byte[] {
(byte) tsdb.getRollupConfig().getIdForAggregator(
rollup_query.getRollupAgg().toString())
})));
@@ -1457,10 +1527,10 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException {
(byte) tsdb.getRollupConfig().getIdForAggregator("sum")
})));
filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL,
- new BinaryPrefixComparator(new byte[] {
+ new BinaryPrefixComparator(new byte[] {
(byte) tsdb.getRollupConfig().getIdForAggregator("count")
})));
-
+
if (existing != null) {
final List combined = new ArrayList(2);
combined.add(existing);
@@ -1475,14 +1545,14 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException {
}
/**
- * Identify the table to be scanned based on the roll up and pre-aggregate
+ * Identify the table to be scanned based on the roll up and pre-aggregate
* query parameters
* @return table name as byte array
* @since 2.4
*/
private byte[] tableToBeScanned() {
final byte[] tableName;
-
+
if (RollupQuery.isValidQuery(rollup_query)) {
if (pre_aggregate) {
tableName= rollup_query.getRollupInterval().getGroupbyTable();
@@ -1497,12 +1567,12 @@ else if (pre_aggregate) {
else {
tableName = tsdb.dataTable();
}
-
+
return tableName;
}
-
+
/** Returns the UNIX timestamp from which we must start scanning. */
- private long getScanStartTimeSeconds() {
+ long getScanStartTimeSeconds() {
// Begin with the raw query start time.
long start = getStartTime();
@@ -1510,15 +1580,15 @@ private long getScanStartTimeSeconds() {
if ((start & Const.SECOND_MASK) != 0L) {
start /= 1000L;
}
-
+
// if we have a rollup query, we have different row key start times so find
// the base time from which we need to search
if (rollup_query != null) {
- long base_time = RollupUtils.getRollupBasetime(start,
+ long base_time = RollupUtils.getRollupBasetime(start,
rollup_query.getRollupInterval());
if (rate) {
// scan one row back so we can get the first rate value.
- base_time = RollupUtils.getRollupBasetime(base_time - 1,
+ base_time = RollupUtils.getRollupBasetime(base_time - 1,
rollup_query.getRollupInterval());
}
return base_time;
@@ -1544,24 +1614,25 @@ private long getScanStartTimeSeconds() {
}
/** Returns the UNIX timestamp at which we must stop scanning. */
- private long getScanEndTimeSeconds() {
+ @VisibleForTesting
+ protected long getScanEndTimeSeconds() {
// Begin with the raw query end time.
long end = getEndTime();
// Convert to seconds if we have a query in ms.
if ((end & Const.SECOND_MASK) != 0L) {
end /= 1000L;
- if (end - (end * 1000) < 1) {
+ if (end == 0) {
// handle an edge case where a user may request a ms time between
// 0 and 1 seconds. Just bump it a second.
end++;
}
}
-
+
if (rollup_query != null) {
- return RollupUtils.getRollupBasetime(end +
- (rollup_query.getRollupInterval().getIntervalSeconds() *
- rollup_query.getRollupInterval().getIntervals()),
+ return RollupUtils.getRollupBasetime(end +
+ (rollup_query.getRollupInterval().getIntervalSeconds() *
+ rollup_query.getRollupInterval().getIntervals()),
rollup_query.getRollupInterval());
}
@@ -1677,9 +1748,6 @@ public void transformDownSamplerToRollupQuery(final Aggregator group_by,
rollup_query = new RollupQuery(best_match_rollups.remove(0),
downsampler.getFunction(), downsampler.getInterval(),
group_by);
- if (group_by == Aggregators.COUNT) {
- aggregator = Aggregators.SUM;
- }
}
catch (NoSuchRollupForIntervalException nre) {
LOG.error("There is no such rollup for the downsample interval "
@@ -1825,6 +1893,14 @@ public int compare(final byte[] a, final byte[] b) {
}
+ RateOptions getRateOptions() { return rate_options; }
+ boolean isRate() { return rate; }
+ Aggregator getAggregator() {return aggregator; }
+ DownsamplingSpecification getDownsampler() { return downsampler; }
+ RollupQuery getRollupQuery() { return rollup_query; }
+ int getQueryIndex() { return query_index; }
+
+
/** Helps unit tests inspect private methods. */
@VisibleForTesting
static class ForTesting {
diff --git a/src/create_table.sh b/src/create_table.sh
index 1cbe666319..917a4ff3de 100755
--- a/src/create_table.sh
+++ b/src/create_table.sh
@@ -23,7 +23,7 @@ COMPRESSION=`echo "$COMPRESSION" | tr a-z A-Z`
# This can save a lot of storage space.
DATA_BLOCK_ENCODING=${DATA_BLOCK_ENCODING-'DIFF'}
DATA_BLOCK_ENCODING=`echo "$DATA_BLOCK_ENCODING" | tr a-z A-Z`
-TSDB_TTL=${TSDB_TTL-'FOREVER'}
+TSDB_TTL=${TSDB_TTL-'2147483647'}
case $COMPRESSION in
(NONE|LZO|GZIP|SNAPPY) :;; # Known good.
diff --git a/src/logback.xml b/src/logback.xml
index ff97a50889..49eff6d58e 100644
--- a/src/logback.xml
+++ b/src/logback.xml
@@ -63,6 +63,9 @@
+
diff --git a/src/query/QueryUtil.java b/src/query/QueryUtil.java
index 203616bd22..bf19f2e46b 100644
--- a/src/query/QueryUtil.java
+++ b/src/query/QueryUtil.java
@@ -15,6 +15,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
@@ -26,13 +27,18 @@
import net.opentsdb.uid.UniqueId;
import org.hbase.async.Bytes;
-import org.hbase.async.FilterList;
import org.hbase.async.FuzzyRowFilter;
+import org.hbase.async.FuzzyRowFilter.FuzzyFilterPair;
import org.hbase.async.KeyRegexpFilter;
import org.hbase.async.Bytes.ByteMap;
+import org.hbase.async.FilterList.Operator;
+import org.hbase.async.FilterList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.hbase.async.ScanFilter;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
import org.hbase.async.Scanner;
/**
@@ -175,9 +181,223 @@ public static String getRowKeyUIDRegex(
return buf.toString();
}
+ /**
+ * Crafts a regular expression for scanning over data table rows and filtering
+ * time series that the user doesn't want.
+ * @param row_key_literals An optional list of key value pairs to filter on.
+ * May be null.
+ * @param explicit_tags Whether or not explicit tags are enabled so that the
+ * regex only picks out series with the specified tags
+ * @return A regular expression string to pass to the storage layer.
+ */
+ private static String getRowKeyUIDRegex(
+ final ByteMap row_key_literals,
+ final boolean explicit_tags) {
+ final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() +
+ Const.TIMESTAMP_BYTES;
+ final short name_width = TSDB.tagk_width();
+ final short value_width = TSDB.tagv_width();
+ final short tagsize = (short) (name_width + value_width);
+ // Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 }
+ // and { 4 5 6 9 8 7 }, the regexp will be:
+ // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$"
+ final StringBuilder buf = new StringBuilder(
+ 15 // "^.{N}" + "(?:.{M})*" + "$"
+ + ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E"
+ * ((row_key_literals == null ? 0 : row_key_literals.size()))));
+
+ // Alright, let's build this regexp. From the beginning...
+ buf.append("(?s)" // Ensure we use the DOTALL flag.
+ + "^.{")
+ // ... start by skipping the salt, metric ID and timestamp.
+ .append(prefix_width)
+ .append("}");
+
+ final Iterator> it = row_key_literals == null ?
+ new ByteMap().iterator() : row_key_literals.iterator();
+
+ while(it.hasNext()) {
+ Entry entry = it.hasNext() ? it.next() : null;
+ // TODO - This look ahead may be expensive. We need to get some data around
+ // whether it's faster for HBase to scan with a look ahead or simply pass
+ // the rows back to the TSD for filtering.
+ final boolean not_key =
+ entry.getValue() != null && entry.getValue().length == 0;
+
+ // Skip any number of tags.
+ if (!explicit_tags) {
+ buf.append("(?:.{").append(tagsize).append("})*");
+ }
+
+ if (not_key) {
+ // start the lookahead as we have a key we explicitly do not want in the
+ // results
+ buf.append("(?!");
+ }
+ buf.append("\\Q");
+
+ addId(buf, entry.getKey(), true);
+ if (entry.getValue() != null && entry.getValue().length > 0) { // Add a group_by.
+ // We want specific IDs. List them: /(AAA|BBB|CCC|..)/
+ buf.append("(?:");
+ for (final byte[] value_id : entry.getValue()) {
+ if (value_id == null) {
+ continue;
+ }
+ buf.append("\\Q");
+ addId(buf, value_id, true);
+ buf.append('|');
+ }
+ // Replace the pipe of the last iteration.
+ buf.setCharAt(buf.length() - 1, ')');
+ } else {
+ buf.append(".{").append(value_width).append('}'); // Any value ID.
+ }
+
+ if (not_key) {
+ // be sure to close off the look ahead
+ buf.append(")");
+ }
+ }
+
+ // Skip any number of tags before the end.
+ if (!explicit_tags) {
+ buf.append("(?:.{").append(tagsize).append("})*");
+ }
+ buf.append("$");
+ return buf.toString();
+ }
+
+ /**
+ * Crafts a list of FuzzyFilters for scanning over data table rows and
+ * filtering time series that the user doesn't want.
+ * Note: The caller has to restrict the scan to proper start and stop
+ * for the filter to work correctly.
+ * @param row_key_literals A list of key value pairs to filter on.
+ * @param fuzzy_key The starting row key we'll adjust for proper filtering.
+ * @return A sorted, non-empty list of FuzzyFilterPair
+ */
+ private static List buildFuzzyFilters(
+ final ByteMap row_key_literals,
+ final byte[] fuzzy_key) {
+ final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() +
+ Const.TIMESTAMP_BYTES;
+ final short name_width = TSDB.tagk_width();
+ final short value_width = TSDB.tagv_width();
+ final short tag_width = (short) (name_width + value_width);
+ int row_key_size = prefix_width;
+ if (row_key_literals != null) {
+ for(byte[][] v: row_key_literals.values()) {
+ final boolean not_key = v!=null && v.length==0;
+ if (!not_key) {
+ row_key_size += tag_width;
+ }
+ }
+ }
+ final List fuzzy_filter_pairs =
+ new ArrayList(row_key_literals.size());
+
+ // Initialize first_fuzzy_key and first_fuzzy_mask
+ // these will serve as model for the fuzzy filter list
+ // generated for tags with multiple values (|)
+ byte[] first_fuzzy_key = Arrays.copyOf(fuzzy_key, fuzzy_key.length);
+ byte[] first_fuzzy_mask = new byte[fuzzy_key.length];
+ int fuzzy_offset = 0;
+
+ // TODO - see if it's less expensive to skip the salt, timestamp and metric.
+ // skip salt & timestamp (filtering should be done by start/stop
+ // of the scanner)
+ while(fuzzy_offset < prefix_width) {
+ first_fuzzy_key[fuzzy_offset] = 0;
+ first_fuzzy_mask[fuzzy_offset++] =
+ (row_key_literals != null) ? (byte)1 : (byte)0;
+ }
+
+ // first pass to build the key and mask
+ Iterator> it = row_key_literals.iterator();
+ while(it.hasNext()) {
+ Entry entry = it.next();
+ final boolean not_key =
+ entry.getValue() != null && entry.getValue().length == 0;
+
+ if (!not_key) {
+ final byte[] tag_key = entry.getKey();
+ System.arraycopy(tag_key, 0,
+ first_fuzzy_key, fuzzy_offset, name_width);
+ for (int i=0; i 0) {
+ tag_value = entry.getValue()[0];
+ } else {
+ tag_value = null;
+ }
+
+ if (tag_value!=null) {
+ System.arraycopy(tag_value, 0,
+ first_fuzzy_key, fuzzy_offset, value_width);
+ for (int i=0; i skip
+ for (int i=0; i entry = it.next();
+ fuzzy_offset += name_width;
+
+ // if multiple values value, generate a new combination of filters
+ // for each value
+ if (entry.getValue()!=null && entry.getValue().length > 1) {
+ for (int i=1; i {
+ @Override
+ public int compare(FuzzyFilterPair pair1, FuzzyFilterPair pair2) {
+ return Bytes.memcmp(pair2.getRowKey(), pair1.getRowKey());
+ }
+ }
+ private static FuzzyFilterComparator FUZZY_FILTER_CMP = new FuzzyFilterComparator();
+
/**
* Sets a filter or filter list on the scanner based on whether or not the
* query had tags it needed to match.
+ * NOTE: This method will sort the group bys.
* @param scanner The scanner to modify.
* @param group_bys An optional list of tag keys that we want to group on. May
* be null.
@@ -205,56 +425,72 @@ public static void setDataTableScanFilter(
return;
}
+ if (group_bys != null) {
+ Collections.sort(group_bys, Bytes.MEMCMP);
+ }
+
final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() +
Const.TIMESTAMP_BYTES;
- final short name_width = TSDB.tagk_width();
- final short value_width = TSDB.tagv_width();
- final byte[] fuzzy_key;
- final byte[] fuzzy_mask;
- if (explicit_tags && enable_fuzzy_filter) {
- fuzzy_key = new byte[prefix_width + (row_key_literals.size() *
- (name_width + value_width))];
- fuzzy_mask = new byte[prefix_width + (row_key_literals.size() *
- (name_width + value_width))];
+
+ final FuzzyRowFilter fuzzy_filter;
+ if (explicit_tags &&
+ enable_fuzzy_filter &&
+ row_key_literals != null &&
+ !row_key_literals.isEmpty()) {
+
+ final byte[] fuzzy_key = new byte[prefix_width + (row_key_literals.size() *
+ (TSDB.tagk_width() + TSDB.tagv_width()))];
System.arraycopy(scanner.getCurrentKey(), 0, fuzzy_key, 0,
scanner.getCurrentKey().length);
+
+ final List fuzzy_filter_pairs =
+ buildFuzzyFilters(row_key_literals, fuzzy_key);
+
+ // The Fuzzy Filter list is sorted: the first and last filters row key
+ // can be used to build the stop key for the scanner
+ final byte[] stop_key = Arrays.copyOf(
+ fuzzy_filter_pairs.get(fuzzy_filter_pairs.size() - 1).getRowKey(),
+ fuzzy_key.length);
+ System.arraycopy(scanner.getCurrentKey(), 0, stop_key, 0, prefix_width);
+ Internal.setBaseTime(stop_key, end_time);
+ int idx = prefix_width + TSDB.tagk_width();
+ // max out the tag values
+ while (idx < stop_key.length) {
+ for (int i = 0; i < TSDB.tagv_width(); i++) {
+ stop_key[idx++] = (byte) 0xFF;
+ }
+ idx += TSDB.tagk_width();
+ }
+
+ scanner.setStartKey(fuzzy_key);
+ scanner.setStopKey(stop_key);
+ fuzzy_filter = new FuzzyRowFilter(fuzzy_filter_pairs);
} else {
- fuzzy_key = fuzzy_mask = null;
+ fuzzy_filter = null;
}
- final String regex = getRowKeyUIDRegex(group_bys, row_key_literals,
- explicit_tags, fuzzy_key, fuzzy_mask);
- final KeyRegexpFilter regex_filter = new KeyRegexpFilter(
- regex.toString(), Const.ASCII_CHARSET);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Regex for scanner: " + scanner + ": " +
- byteRegexToString(regex));
+ final String regex = getRowKeyUIDRegex(row_key_literals, explicit_tags);
+ final KeyRegexpFilter regex_filter;
+ if (!Strings.isNullOrEmpty(regex)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Regex for scanner: " + scanner + ": " +
+ byteRegexToString(regex));
+ }
+ regex_filter = new KeyRegexpFilter(regex.toString(),
+ Const.ASCII_CHARSET);
+ } else {
+ regex_filter = null;
}
- if (!(explicit_tags && enable_fuzzy_filter)) {
+ if (fuzzy_filter != null && !Strings.isNullOrEmpty(regex)) {
+ final FilterList filter = new FilterList(Lists.newArrayList(fuzzy_filter,
+ regex_filter),Operator.MUST_PASS_ALL);
+ scanner.setFilter(filter);
+ } else if (fuzzy_filter != null) {
+ scanner.setFilter(fuzzy_filter);
+ } else if (!Strings.isNullOrEmpty(regex)) {
scanner.setFilter(regex_filter);
- return;
}
-
- scanner.setStartKey(fuzzy_key);
- final byte[] stop_key = Arrays.copyOf(fuzzy_key, fuzzy_key.length);
- Internal.setBaseTime(stop_key, end_time);
- int idx = Const.SALT_WIDTH() + TSDB.metrics_width() +
- Const.TIMESTAMP_BYTES + TSDB.tagk_width();
- // max out the tag values
- while (idx < stop_key.length) {
- for (int i = 0; i < TSDB.tagv_width(); i++) {
- stop_key[idx++] = (byte) 0xFF;
- }
- idx += TSDB.tagk_width();
- }
- scanner.setStopKey(stop_key);
- final List filters = new ArrayList(2);
- filters.add(
- new FuzzyRowFilter(
- new FuzzyRowFilter.FuzzyFilterPair(fuzzy_key, fuzzy_mask)));
- filters.add(regex_filter);
- scanner.setFilter(new FilterList(filters));
}
/**
@@ -403,4 +639,24 @@ public static String byteRegexToString(final String regexp) {
}
return buf.toString();
}
+
+ /**
+ * Compares two timestamps where either can be in seconds or milliseconds.
+ *
+ * @param ts1 The first timestamp in either seconds or milliseconds.
+ * @param ts2 The second timestamp in either seconds or milliseconds.
+ * @return Whether the first timestamp is after the second
+ */
+ public static boolean isTimestampAfter(long ts1, long ts2) {
+ boolean ts1InSeconds = (ts1 & Const.SECOND_MASK) == 0;
+ boolean ts2InSeconds = (ts2 & Const.SECOND_MASK) == 0;
+
+ if (ts1InSeconds && !ts2InSeconds) {
+ ts1 *= 1000L;
+ } else if (!ts1InSeconds && ts2InSeconds) {
+ ts2 *= 1000L;
+ }
+
+ return ts1 > ts2;
+ }
}
diff --git a/src/query/expression/ExpressionFactory.java b/src/query/expression/ExpressionFactory.java
index e0fbdd44e8..43358e6eb6 100644
--- a/src/query/expression/ExpressionFactory.java
+++ b/src/query/expression/ExpressionFactory.java
@@ -37,6 +37,7 @@ public final class ExpressionFactory {
available_functions.put("highestMax", new HighestMax());
available_functions.put("shift", new TimeShift());
available_functions.put("timeShift", new TimeShift());
+ available_functions.put("firstDiff", new FirstDifference());
}
/** Don't instantiate me! */
diff --git a/src/query/expression/FirstDifference.java b/src/query/expression/FirstDifference.java
new file mode 100644
index 0000000000..6dc75bc088
--- /dev/null
+++ b/src/query/expression/FirstDifference.java
@@ -0,0 +1,101 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2015 The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version. This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details. You should have received a copy
+// of the GNU Lesser General Public License along with this program. If not,
+// see query_results, final List |