forked from zendesk/maxwell
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'track-nontx-inflight-messesages-with-backpressure'
- Loading branch information
Showing
4 changed files
with
122 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 81 additions & 25 deletions
106
src/main/java/com/zendesk/maxwell/producer/InflightMessageList.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,60 +1,116 @@ | ||
package com.zendesk.maxwell.producer; | ||
/* respresents a list of inflight messages -- stuff being sent over the | ||
network, that may complete in any order. Allows for only bumping | ||
the binlog position upon completion of the oldest outstanding item. | ||
/* | ||
respresents a list of inflight messages -- stuff being sent over the | ||
network, that may complete in any order. Allows for only bumping | ||
the binlog position upon completion of the oldest outstanding item. | ||
Assumes .addInflight(position) will be call monotonically. | ||
*/ | ||
Assumes .addInflight(position) will be call monotonically. | ||
*/ | ||
|
||
import com.zendesk.maxwell.replication.BinlogPosition; | ||
|
||
import java.util.concurrent.locks.Lock; | ||
import java.util.concurrent.locks.ReentrantLock; | ||
import java.util.concurrent.locks.Condition; | ||
import java.util.HashSet; | ||
import java.util.LinkedHashMap; | ||
import java.util.Iterator; | ||
|
||
public class InflightMessageList { | ||
class InflightMessage { | ||
class InflightTXMessage { | ||
public final BinlogPosition position; | ||
public boolean isComplete; | ||
InflightMessage(BinlogPosition position) { | ||
|
||
public InflightTXMessage(BinlogPosition position) { | ||
this.position = position; | ||
this.isComplete = false; | ||
} | ||
} | ||
|
||
private LinkedHashMap<String, InflightMessage> linkedMap; | ||
private final HashSet<Integer> nonTXMessages; | ||
private final LinkedHashMap<Integer, InflightTXMessage> txMessages; | ||
|
||
private final Lock txLock = new ReentrantLock(); | ||
private final Lock nonTXLock = new ReentrantLock(); | ||
private final Condition nonTXMessagesNotFull = nonTXLock.newCondition(); | ||
|
||
private final int MAX_INFLIGHT_NON_TX_MESSAGES = 10000; | ||
|
||
public InflightMessageList() { | ||
this.linkedMap = new LinkedHashMap<>(); | ||
this.nonTXMessages = new HashSet<Integer>(); | ||
this.txMessages = new LinkedHashMap<Integer, InflightTXMessage>(); | ||
} | ||
|
||
public synchronized void addMessage(BinlogPosition p) { | ||
InflightMessage m = new InflightMessage(p); | ||
this.linkedMap.put(p.toString(), m); | ||
public void addTXMessage(int rowId, BinlogPosition position) { | ||
txLock.lock(); | ||
|
||
try { | ||
InflightTXMessage m = new InflightTXMessage(position); | ||
txMessages.put(rowId, m); | ||
} finally { | ||
txLock.unlock(); | ||
} | ||
|
||
return; | ||
} | ||
|
||
/* returns the position that stuff is complete up to, or null if there were no changes */ | ||
public synchronized BinlogPosition completeMessage(BinlogPosition p) { | ||
InflightMessage m = this.linkedMap.get(p.toString()); | ||
assert(m != null); | ||
public void addNonTXMessage(int rowId) throws InterruptedException { | ||
nonTXLock.lock(); | ||
|
||
try { | ||
while(nonTXMessages.size() > MAX_INFLIGHT_NON_TX_MESSAGES) { | ||
nonTXMessagesNotFull.await(); | ||
} | ||
|
||
m.isComplete = true; | ||
this.nonTXMessages.add(rowId); | ||
} finally { | ||
nonTXLock.unlock(); | ||
} | ||
} | ||
|
||
/* returns the position that stuff is complete up to, or null if there were no changes */ | ||
public BinlogPosition completeTXMessage(int rowId) { | ||
BinlogPosition completeUntil = null; | ||
Iterator<InflightMessage> iterator = this.linkedMap.values().iterator(); | ||
|
||
while ( iterator.hasNext() ) { | ||
InflightMessage msg = iterator.next(); | ||
if ( !msg.isComplete ) | ||
break; | ||
txLock.lock(); | ||
|
||
try { | ||
InflightTXMessage m = txMessages.get(rowId); | ||
assert(m != null); | ||
|
||
m.isComplete = true; | ||
Iterator<InflightTXMessage> iterator = txMessages.values().iterator(); | ||
|
||
while ( iterator.hasNext() ) { | ||
InflightTXMessage msg = iterator.next(); | ||
if ( !msg.isComplete ) | ||
break; | ||
|
||
completeUntil = msg.position; | ||
iterator.remove(); | ||
} | ||
|
||
completeUntil = msg.position; | ||
iterator.remove(); | ||
} finally { | ||
txLock.unlock(); | ||
} | ||
|
||
return completeUntil; | ||
} | ||
|
||
public void completeNonTXMessage(int rowId) { | ||
nonTXLock.lock(); | ||
|
||
try { | ||
nonTXMessages.remove(rowId); | ||
nonTXMessagesNotFull.signal(); | ||
} finally { | ||
nonTXLock.unlock(); | ||
} | ||
|
||
return; | ||
} | ||
|
||
public int size() { | ||
return linkedMap.size(); | ||
return txMessages.size() + nonTXMessages.size(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters