Screenshot CC BY-NC 3.0 Before: the mess CC BY-NC 3.0 Before: - - PowerPoint PPT Presentation

🚬 🚬

CC BY-NC 3.0

Screenshot

CC BY-NC 3.0

Before: the mess

CC BY-NC 3.0

Before: the mess

CC BY-NC 3.0

Before: the mess

Cassandra Akka & Spray C++ & CUDA RabbitMQ

CC BY-NC 3.0

Before: the mess

scene topic identity topic (Map[String, String], Array[Byte]) (Map[String, String], Array[Byte])

CC BY-NC 3.0

Before: the mess

_tmp_x topics JSON scene topic identity topic (Map[String, String], Array[Byte]) (Map[String, String], Array[Byte])

CC BY-NC 3.0

Before: the mess

fire-and-forget at-most-once non-durable fire-and-forget at-most-once

CC BY-NC 3.0

Asynchronous request–response orchestration

class Orchestrator extends Actor with ActorFSM[Orchestrator.State, Orchestrator.Data] { startWith(Idle, UninitializedData) when(Idle, idleTimeout)(idleSF) when(ImageProcessing, stepTimeout)(imageProcessingSF) when(WaitingForProcessingResult, stepTimeout)(waitingForProcessingSF) whenUnhandled(timeoutSF)

• nTransition {

case _ -> Aborted => ??? ... } def idleSF: StateFunction = ??? def imageProcessingSF: StateFunction = ??? def waitingForProcessingSF: StateFunction = ??? def timeoutSF: StateFunction = { case Event(StateTimeout, data: RunningTransactionData) => goto(Aborted) } }

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

Akka Kafka Lagom Cassandra

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

tweet-image topic identity group text group scene topic identity topic scene group text topic

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

tweet-image topic scene topic identity topic text topic message Scene { … } message Identity { … } bytes image; message Envelope { int32 version = 1; string handle = 2; int64 processingTimestamp = 3; int64 ingestionTimestamp = 4; string correlationId = 5; string messageId = 6; string messageType = 7; bytes payload = 8; } message Text { … }

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

at-least-once at-least-once at-most-once fire-and-forget at-least-once

CC BY-NC 3.0

Persistence and formats

message Envelope {
 int32 version = 1;
 string handle = 2;
 int64 processingTimestamp = 3;
 int64 ingestionTimestamp = 4;
 string correlationId = 5;
 string messageId = 6;
 string messageType = 7;
 bytes payload = 8;
 }

CC BY-NC 3.0

Persistence and formats

message Text { repeated string areas = 1;
 }

CC BY-NC 3.0

Persistence and formats

message Scene { message Label {
 string label = 1;
 double score = 2;
 } 
 repeated Label labels = 3;
 }

CC BY-NC 3.0

Persistence and formats

message Identity {


• neof face {


IdentifiedFace identifiedFace = 1;
 UnknownFace unknownFace = 2;
 }
 
 message IdentifiedFace {
 string name = 1;
 double score = 2;
 }
 
 message UnknownFace { }
 }

CC BY-NC 3.0

Persistence and formats

message IdentifyFace {
 int64 ingestionTimestamp = 2;
 string correlationId = 3;
 string handle = 4;
 bytes image = 5;
 }
 
 message IdentifyFaces {
 repeated IdentifyFace identifyFaces = 1;
 }
 
 message FaceImage {
 double confidence = 1;
 int32 x = 2;
 int32 y = 3;
 int32 w = 4;
 int32 h = 5;
 bytes rgbBitmap = 6;
 }

CC BY-NC 3.0

Fire–and–forget send

• bject Act {

def props(config: Config): Props = { val producerConf = KafkaProducer.Conf(config.getConfig("..."), new StringSerializer, KafkaSerializer[Envelope](_.toByteArray)) Props(classOf[Act], producerConf) } } class Act(producerConf: KafkaProducer.Conf[String, Envelope]) extends Actor { private[this] val producer = KafkaProducer(conf = producerConf)

• verride def receive: Receive = {

case TweetImage(handle, content) => producer.send(KafkaProducerRecord("tweet-image", handle, Envelope(version = 100, handle = handle, ingestionTimestamp = System.nanoTime(), processingTimestamp = System.nanoTime(), messageId = UUID.randomUUID().toString, correlationId = UUID.randomUUID().toString, payload = content))) } }

CC BY-NC 3.0

At least once delivery I

class SceneClassifierActor(…) extends Actor { private[this] val kafkaConsumerActor = context.actorOf(…) private[this] val producer = KafkaProducer(…)

• verride def receive: Receive = {

case extractor(consumerRecords) => val futures = consumerRecords.pairs.flatMap { case (_, envelope) => val outEnvelope = … Some(producer.send(KafkaProducerRecord("scene", envelope.handle, outEnvelope))) } import context.dispatcher Future.sequence(futures).onSuccess { case _ => kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true) } } }

CC BY-NC 3.0

At least once delivery II

class IdentityMatcherActor(...) extends PersistentActor with AtLeastOnceDelivery {

• verride val persistenceId: String = "identity-matcher-actor"

def identifyFacesAndSend(identifyFaces: Seq[IdentifyFace])(implicit executor: ExecutionContext): Future[Unit] = { // Future.sequence(producer.send(...)) } def handleIdentifyFace: Receive = { case (deliveryId: Long, identifyFaces: IdentifyFaces) => import context.dispatcher identifyFacesAndSend(identifyFaces.identifyFaces).onSuccess { case _ => confirmDelivery(deliveryId) } case IdentifyFaces(faces) => import context.dispatcher identifyFacesAndSend(faces).onFailure { case _ => self ! Kill } }

• verride def receiveRecover: Receive = handleIdentifyFace
• verride def receiveCommand: Receive = handleIdentifyFace orElse {

case extractor(consumerRecords) => val identifyFaces = consumerRecords.pairs.map { case (_, envelope) => IdentifyFace(envelope.ingestionTimestamp, envelope.correlationId, envelope.handle, envelope.payload)) } persist(IdentifyFaces(identifyFaces = identifyFaces)) { result => deliver(self.path)(deliveryId => (deliveryId, result)) sender() ! Confirm(consumerRecords.offsets, commit = true) } } }

CC BY-NC 3.0

At least once delivery III

public interface TweetImageService extends Service {
 Topic<Envelope> tweetImageTopic();
 …
 }
 public class TextServiceImpl implements TextService {
 private final PersistentEntityRegistry persistentEntityRegistry;
 
 public Topic<Envelope> textTopic() {
 return TopicProducer.singleStreamWithOffset(offset -> persistentEntityRegistry
 .eventStream(TextEntityEvent.OcredTag.INSTANCE, offset)
 .map(p -> new Pair<>(p.first().envelope(), offset))
 );
 }
 
 @Inject
 public TextServiceImpl(PersistentEntityRegistry persistentEntityRegistry, TweetImageService tweetImageService) {
 this.persistentEntityRegistry = persistentEntityRegistry;
 persistentEntityRegistry.register(TextEntity.class);
 tweetImageService.tweetImageTopic().subscribe().withGroupId("text").atLeastOnce(Flow.fromFunction(this::extractText));
 }
 
 private Done extractText(Envelope envelope) {
 TextEntityCommand.Ocr command = new TextEntityCommand.Ocr(…);
 PersistentEntityRef<TextEntityCommand> ref = persistentEntityRegistry.refFor(TextEntity.class, envelope.handle());
 ref.ask(command);
 return Done.getInstance();
 }
 
 }

CC BY-NC 3.0

At most once delivery

class DashboardSinkActor(...) extends Actor { private[this] val kafkaConsumerActor = context.actorOf(...)

• verride def receive: Receive = {

case extractor(consumerRecords) => consumerRecords.pairs.foreach { case (_, envelope) => context.system.eventStream.publish(envelope); } kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true) } }

CC BY-NC 3.0

ENOUGH SLIDES!

CC BY-NC 3.0

After: proper microservices

text identity ingest dashboard scene

@cakesolutions

www.cakesolutions.net

MANCHESTER LONDON NEW YORK

CC BY-NC 3.0

enquiries@cakesolutions.net