Skip to content

Commit 1ec8f30

Browse files
lmagic233lmagic233
committedSep 16, 2021
Add boolean parameter for CUMULATE window TVF to support incremetal-only output
1 parent e397abd commit 1ec8f30

File tree

9 files changed

+87
-23
lines changed

9 files changed

+87
-23
lines changed
 

‎flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/ExecutionConfigOptions.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,13 @@ public class ExecutionConfigOptions {
248248
.withDescription(
249249
"Sets the window elements buffer size limit used in group window agg operator.");
250250

251+
/* @Documentation.TableOption(execMode = Documentation.ExecMode.STREAMING)
252+
public static final ConfigOption<Boolean> TABLE_EXEC_WINDOW_AGG_INCREMENTAL_OUTPUT_ONLY =
253+
key("table.exec.window-agg.incremental-output-only")
254+
.defaultValue(false)
255+
.withDescription(
256+
"Indicates whether to output incrementally aggregated results when using shared-slice windows (e.g. HOP, CUMULATE)."); */
257+
251258
// ------------------------------------------------------------------------
252259
// Lookup Options
253260
// ------------------------------------------------------------------------

‎flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/SqlCumulateTableFunction.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,19 @@ private static class OperandMetadataImpl extends AbstractOperandMetadata {
4545
OperandMetadataImpl() {
4646
super(
4747
ImmutableList.of(
48-
PARAM_DATA, PARAM_TIMECOL, PARAM_STEP, PARAM_SIZE, PARAM_OFFSET),
49-
4);
48+
PARAM_DATA, PARAM_TIMECOL, PARAM_STEP, PARAM_SIZE, PARAM_INCREMENTAL, PARAM_OFFSET),
49+
5);
5050
}
5151

5252
@Override
5353
public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) {
5454
if (!checkTableAndDescriptorOperands(callBinding, 1)) {
5555
return throwValidationSignatureErrorOrReturnFalse(callBinding, throwOnFailure);
5656
}
57-
if (!checkIntervalOperands(callBinding, 2)) {
57+
if (!checkIntervalOperands(callBinding, 2, callBinding.getOperandCount() - 1)) {
5858
return throwValidationSignatureErrorOrReturnFalse(callBinding, throwOnFailure);
5959
}
60-
if (callBinding.getOperandCount() == 5) {
60+
if (callBinding.getOperandCount() == 6) {
6161
return throwValidationSignatureErrorOrReturnFalse(callBinding, throwOnFailure);
6262
}
6363
// check time attribute
@@ -69,7 +69,7 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail
6969
public String getAllowedSignatures(SqlOperator op, String opName) {
7070
return opName
7171
+ "(TABLE table_name, DESCRIPTOR(timecol), "
72-
+ "datetime interval, datetime interval)";
72+
+ "datetime interval, datetime interval, <BOOLEAN>)";
7373
}
7474
}
7575
}

‎flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/SqlWindowTableFunction.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ public class SqlWindowTableFunction extends SqlFunction implements SqlTableFunct
8383
/** The slide interval, only used for HOP window. */
8484
protected static final String PARAM_STEP = "STEP";
8585

86+
/** Whether to output incremental result with each slice, only used for CUMULATE window. */
87+
protected static final String PARAM_INCREMENTAL = "INCREMENTAL";
88+
8689
/**
8790
* Type-inference strategy whereby the row type of a table function call is a ROW, which is
8891
* combined from the row type of operand #0 (which is a TABLE) and two additional fields. The
@@ -305,8 +308,12 @@ Optional<RuntimeException> checkTimeColumnDescriptorOperand(
305308
* @return true if validation passes
306309
*/
307310
boolean checkIntervalOperands(SqlCallBinding callBinding, int startPos) {
311+
return checkIntervalOperands(callBinding, startPos, callBinding.getOperandCount());
312+
}
313+
314+
boolean checkIntervalOperands(SqlCallBinding callBinding, int startPos, int endPos) {
308315
final SqlValidator validator = callBinding.getValidator();
309-
for (int i = startPos; i < callBinding.getOperandCount(); i++) {
316+
for (int i = startPos; i < endPos; i++) {
310317
final RelDataType type = validator.getValidatedNodeType(callBinding.operand(i));
311318
if (!SqlTypeUtil.isInterval(type)) {
312319
return false;

‎flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/logical/CumulativeWindowSpec.java

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,26 +33,32 @@
3333
public class CumulativeWindowSpec implements WindowSpec {
3434
public static final String FIELD_NAME_MAX_SIZE = "maxSize";
3535
public static final String FIELD_NAME_STEP = "step";
36+
public static final String FIELD_NAME_INCREMENTAL = "incremental";
3637

3738
@JsonProperty(FIELD_NAME_MAX_SIZE)
3839
private final Duration maxSize;
3940

4041
@JsonProperty(FIELD_NAME_STEP)
4142
private final Duration step;
4243

44+
@JsonProperty(FIELD_NAME_INCREMENTAL)
45+
private final Boolean incremental;
46+
4347
@JsonCreator
4448
public CumulativeWindowSpec(
4549
@JsonProperty(FIELD_NAME_MAX_SIZE) Duration maxSize,
46-
@JsonProperty(FIELD_NAME_STEP) Duration step) {
50+
@JsonProperty(FIELD_NAME_STEP) Duration step,
51+
@JsonProperty(FIELD_NAME_INCREMENTAL) Boolean incremental) {
4752
this.maxSize = checkNotNull(maxSize);
4853
this.step = checkNotNull(step);
54+
this.incremental = checkNotNull(incremental);
4955
}
5056

5157
@Override
5258
public String toSummaryString(String windowing) {
5359
return String.format(
54-
"CUMULATE(%s, max_size=[%s], step=[%s])",
55-
windowing, formatWithHighestUnit(maxSize), formatWithHighestUnit(step));
60+
"CUMULATE(%s, max_size=[%s], step=[%s], incremental=[%s])",
61+
windowing, formatWithHighestUnit(maxSize), formatWithHighestUnit(step), incremental);
5662
}
5763

5864
public Duration getMaxSize() {
@@ -63,6 +69,10 @@ public Duration getStep() {
6369
return step;
6470
}
6571

72+
public Boolean getIncremental() {
73+
return incremental;
74+
}
75+
6676
@Override
6777
public boolean equals(Object o) {
6878
if (this == o) {
@@ -72,18 +82,18 @@ public boolean equals(Object o) {
7282
return false;
7383
}
7484
CumulativeWindowSpec that = (CumulativeWindowSpec) o;
75-
return maxSize.equals(that.maxSize) && step.equals(that.step);
85+
return maxSize.equals(that.maxSize) && step.equals(that.step) && incremental.equals(that.incremental);
7686
}
7787

7888
@Override
7989
public int hashCode() {
80-
return Objects.hash(CumulativeWindowSpec.class, maxSize, step);
90+
return Objects.hash(CumulativeWindowSpec.class, maxSize, step, incremental);
8191
}
8292

8393
@Override
8494
public String toString() {
8595
return String.format(
86-
"CUMULATE(max_size=[%s], step=[%s])",
87-
formatWithHighestUnit(maxSize), formatWithHighestUnit(step));
96+
"CUMULATE(max_size=[%s], step=[%s], incremental=[%s])",
97+
formatWithHighestUnit(maxSize), formatWithHighestUnit(step), incremental);
8898
}
8999
}

‎flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregateBase.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,15 @@ protected SliceAssigner createSliceAssigner(
117117
} else if (windowSpec instanceof CumulativeWindowSpec) {
118118
Duration maxSize = ((CumulativeWindowSpec) windowSpec).getMaxSize();
119119
Duration step = ((CumulativeWindowSpec) windowSpec).getStep();
120+
Boolean incremental = ((CumulativeWindowSpec) windowSpec).getIncremental();
120121
if (maxSize.toMillis() % step.toMillis() != 0) {
121122
throw new TableException(
122123
String.format(
123124
"CUMULATE table function based aggregate requires maxSize must be an "
124125
+ "integral multiple of step, but got maxSize %s ms and step %s ms",
125126
maxSize.toMillis(), step.toMillis()));
126127
}
127-
return SliceAssigners.cumulative(timeAttributeIndex, shiftTimeZone, maxSize, step);
128+
return SliceAssigners.cumulative(timeAttributeIndex, shiftTimeZone, maxSize, step, incremental);
128129

129130
} else {
130131
throw new UnsupportedOperationException(windowSpec + " is not supported yet.");

‎flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
package org.apache.flink.table.planner.plan.utils
2020

2121
import org.apache.flink.table.api.{DataTypes, TableConfig, TableException, ValidationException}
22-
import org.apache.flink.table.planner.JBigDecimal
22+
import org.apache.flink.table.planner.{JBigDecimal, JBoolean}
2323
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
2424
import org.apache.flink.table.planner.expressions._
2525
import org.apache.flink.table.planner.functions.sql.{FlinkSqlOperatorTable, SqlWindowTableFunction}
@@ -31,7 +31,6 @@ import org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_
3131
import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.fromDataTypeToLogicalType
3232
import org.apache.flink.table.types.logical.TimestampType
3333
import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.canBeTimeAttributeType
34-
3534
import org.apache.calcite.rel.`type`.RelDataType
3635
import org.apache.calcite.rel.core.{Aggregate, AggregateCall, Calc}
3736
import org.apache.calcite.rex._
@@ -41,7 +40,6 @@ import org.apache.calcite.util.ImmutableBitSet
4140

4241
import java.time.Duration
4342
import java.util.Collections
44-
4543
import scala.collection.JavaConversions._
4644
import scala.collection.mutable.ArrayBuffer
4745

@@ -202,7 +200,8 @@ object WindowUtil {
202200
case FlinkSqlOperatorTable.CUMULATE =>
203201
val step = getOperandAsLong(windowCall.operands(2))
204202
val maxSize = getOperandAsLong(windowCall.operands(3))
205-
new CumulativeWindowSpec(Duration.ofMillis(maxSize), Duration.ofMillis(step))
203+
val incremental = getOperandAsBoolean(windowCall.operands(4))
204+
new CumulativeWindowSpec(Duration.ofMillis(maxSize), Duration.ofMillis(step), incremental)
206205
}
207206

208207
new TimeAttributeWindowingStrategy(windowSpec, timeAttributeType, timeIndex)
@@ -314,4 +313,12 @@ object WindowUtil {
314313
}
315314
}
316315

316+
private def getOperandAsBoolean(operand: RexNode): Boolean = {
317+
operand match {
318+
case v: RexLiteral if v.getTypeName.getFamily == SqlTypeFamily.BOOLEAN =>
319+
v.getValue.asInstanceOf[JBoolean]
320+
case _ => throw new TableException("This operand is of BOOLEAN type only.")
321+
}
322+
}
323+
317324
}

‎flink-table/flink-table-runtime-blink/pom.xml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,19 @@ under the License.
176176
</execution>
177177
</executions>
178178
</plugin>
179+
180+
<plugin>
181+
<groupId>org.apache.maven.plugins</groupId>
182+
<artifactId>maven-source-plugin</artifactId>
183+
<executions>
184+
<execution>
185+
<id>attach-sources</id>
186+
<goals>
187+
<goal>jar</goal>
188+
</goals>
189+
</execution>
190+
</executions>
191+
</plugin>
179192
</plugins>
180193
</build>
181194
</project>

‎flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceSharedWindowAggProcessor.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,17 @@
2424
import org.apache.flink.table.runtime.operators.aggregate.window.buffers.WindowBuffer;
2525
import org.apache.flink.table.runtime.operators.window.slicing.SliceAssigner;
2626
import org.apache.flink.table.runtime.operators.window.slicing.SliceAssigners;
27+
import org.apache.flink.table.runtime.operators.window.slicing.SliceAssigners.CumulativeSliceAssigner;
28+
import org.apache.flink.table.runtime.operators.window.slicing.SliceAssigners.HoppingSliceAssigner;
2729
import org.apache.flink.table.runtime.operators.window.slicing.SliceSharedAssigner;
30+
import org.apache.flink.types.Row;
2831

2932
import javax.annotation.Nullable;
3033

3134
import java.io.Serializable;
3235
import java.time.ZoneId;
36+
import java.util.HashMap;
37+
import java.util.Map;
3338
import java.util.Optional;
3439
import java.util.function.Supplier;
3540

@@ -68,7 +73,15 @@ public void fireWindow(Long windowEnd) throws Exception {
6873
if (!isWindowEmpty()) {
6974
// for hopping windows, the triggered window may be an empty window
7075
// (see register next window below), for such window, we shouldn't emit it
71-
collect(aggResult);
76+
if (sliceSharedAssigner instanceof CumulativeSliceAssigner
77+
&& ((CumulativeSliceAssigner) sliceSharedAssigner).isIncremental()) {
78+
RowData stateValue = windowState.value(windowEnd);
79+
if (stateValue == null || !stateValue.equals(aggResult)) {
80+
collect(aggResult);
81+
}
82+
} else {
83+
collect(aggResult);
84+
}
7285
}
7386

7487
// we should register next window timer here,

‎flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/operators/window/slicing/SliceAssigners.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,9 @@ public static HoppingSliceAssigner hopping(
9191
* @param step the step interval of the generated windows.
9292
*/
9393
public static CumulativeSliceAssigner cumulative(
94-
int rowtimeIndex, ZoneId shiftTimeZone, Duration maxSize, Duration step) {
94+
int rowtimeIndex, ZoneId shiftTimeZone, Duration maxSize, Duration step, Boolean incremental) {
9595
return new CumulativeSliceAssigner(
96-
rowtimeIndex, shiftTimeZone, maxSize.toMillis(), step.toMillis(), 0);
96+
rowtimeIndex, shiftTimeZone, maxSize.toMillis(), step.toMillis(), incremental, 0);
9797
}
9898

9999
/**
@@ -284,17 +284,18 @@ public static final class CumulativeSliceAssigner extends AbstractSliceAssigner
284284
/** Creates a new {@link CumulativeSliceAssigner} with a new specified offset. */
285285
public CumulativeSliceAssigner withOffset(Duration offset) {
286286
return new CumulativeSliceAssigner(
287-
rowtimeIndex, shiftTimeZone, maxSize, step, offset.toMillis());
287+
rowtimeIndex, shiftTimeZone, maxSize, step, incremental, offset.toMillis());
288288
}
289289

290290
private final long maxSize;
291291
private final long step;
292292
private final long offset;
293+
private final boolean incremental;
293294
private final ReusableListIterable reuseToBeMergedList = new ReusableListIterable();
294295
private final ReusableListIterable reuseExpiredList = new ReusableListIterable();
295296

296297
protected CumulativeSliceAssigner(
297-
int rowtimeIndex, ZoneId shiftTimeZone, long maxSize, long step, long offset) {
298+
int rowtimeIndex, ZoneId shiftTimeZone, long maxSize, long step, boolean incremental, long offset) {
298299
super(rowtimeIndex, shiftTimeZone);
299300
if (maxSize <= 0 || step <= 0) {
300301
throw new IllegalArgumentException(
@@ -312,6 +313,7 @@ protected CumulativeSliceAssigner(
312313
this.maxSize = maxSize;
313314
this.step = step;
314315
this.offset = offset;
316+
this.incremental = incremental;
315317
}
316318

317319
@Override
@@ -379,6 +381,10 @@ public Optional<Long> nextTriggerWindow(long windowEnd, Supplier<Boolean> isWind
379381
return Optional.of(nextWindowEnd);
380382
}
381383
}
384+
385+
public boolean isIncremental() {
386+
return incremental;
387+
}
382388
}
383389

384390
/**

0 commit comments

Comments
 (0)
Please sign in to comment.