.
-Jekyll Group-By-Array is distributed under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+```
+{% include video id="MY_VIDEO_ID" provider="youtube" %}
+```
-Minimal Mistakes incorporates [@allejo's Pure Liquid Jekyll Table of Contents](https://allejo.io/blog/a-jekyll-toc-in-liquid-only/),
-Copyright (c) 2017 Vladimir Jimenez.
-Pure Liquid Jekyll Table of Contents is distributed under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+## How to contribute
-Minimal Mistakes incorporates [Lunr](http://lunrjs.com),
-Copyright (c) 2018 Oliver Nightingale.
-Lunr is distributed under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+Submit a PR with your article in the `_posts` or in the `_drafts` directory! We can carry the review/article discussion in the PR code review.
diff --git a/_config.yml b/_config.yml
index 92e645cc2546..07864462f9cc 100644
--- a/_config.yml
+++ b/_config.yml
@@ -6,7 +6,6 @@
# `jekyll serve`. If you change this file, please restart the server process.
# Theme Settings
-#
# Review documentation to determine if you should use `theme` or `remote_theme`
# https://mmistakes.github.io/minimal-mistakes/docs/quick-start-guide/#installing-the-theme
@@ -16,130 +15,150 @@ minimal_mistakes_skin : "default" # "air", "aqua", "contrast", "dark", "dirt"
# Site Settings
locale : "en-US"
-title : "Site Title"
+title : "Rock the JVM Blog"
title_separator : "-"
-subtitle : # site tagline that appears below site title in masthead
-name : "Your Name"
-description : "An amazing website."
-url : # the base hostname & protocol for your site e.g. "https://mmistakes.github.io"
-baseurl : # the subpath of your site, e.g. "/blog"
-repository : # GitHub username/repo-name e.g. "mmistakes/minimal-mistakes"
-teaser : # path of fallback teaser image, e.g. "/assets/images/500x300.png"
-logo : # path of logo image to display in the masthead, e.g. "/assets/images/88x88.png"
-masthead_title : # overrides the website title displayed in the masthead, use " " for no title
+subtitle : "Articles on Scala, Kotlin, software and more"
+name : "Rock the JVM"
+description : "Blog about Scala, functional programming, distributed systems (Akka), big data (Apache Spark) and the JVM."
+url : "https://blog.rockthejvm.com"
+baseurl : "" # the subpath of your site, e.g. "/blog"
+repository : "" # GitHub username/repo-name e.g. "mmistakes/minimal-mistakes"
+teaser : "" # path of fallback teaser image, e.g. "/assets/images/500x300.png"
+# logo : "/assets/fiery-lava 128x128.png" # path of logo image to display in the masthead, e.g. "/assets/images/88x88.png"
+masthead_title : "" # overrides the website title displayed in the masthead, use " " for no title
# breadcrumbs : false # true, false (default)
words_per_minute : 200
comments:
- provider : # false (default), "disqus", "discourse", "facebook", "staticman", "staticman_v2", "utterances", "custom"
+ provider : false # false (default), "disqus", "discourse", "facebook", "staticman", "staticman_v2", "utterances", "custom"
disqus:
- shortname : # https://help.disqus.com/customer/portal/articles/466208-what-s-a-shortname-
+ shortname : "" # https://help.disqus.com/customer/portal/articles/466208-what-s-a-shortname-
discourse:
- server : # https://meta.discourse.org/t/embedding-discourse-comments-via-javascript/31963 , e.g.: meta.discourse.org
+ server : "" # https://meta.discourse.org/t/embedding-discourse-comments-via-javascript/31963 , e.g.: meta.discourse.org
facebook:
# https://developers.facebook.com/docs/plugins/comments
- appid :
- num_posts : # 5 (default)
- colorscheme : # "light" (default), "dark"
+ appid : ""
+ num_posts : 5 # 5 (default)
+ colorscheme : "light" # "light" (default), "dark"
utterances:
- theme : # "github-light" (default), "github-dark"
- issue_term : # "pathname" (default)
+ theme : "github-light" # "github-light" (default), "github-dark"
+ issue_term : "pathname" # "pathname" (default)
staticman:
- branch : # "master"
- endpoint : # "https://{your Staticman v3 API}/v3/entry/github/"
+ branch : "master"
+ endpoint : "" # "https://{your Staticman v3 API}/v3/entry/github/"
reCaptcha:
- siteKey :
- secret :
+ siteKey : ""
+ secret : ""
atom_feed:
- path : # blank (default) uses feed.xml
-search : # true, false (default)
-search_full_content : # true, false (default)
-search_provider : # lunr (default), algolia, google
+ path : "/feed.xml" # blank (default) uses feed.xml
+search : true # true, false (default)
+search_full_content : false # true, false (default)
+search_provider : lunr # lunr (default), algolia, google
algolia:
- application_id : # YOUR_APPLICATION_ID
- index_name : # YOUR_INDEX_NAME
- search_only_api_key : # YOUR_SEARCH_ONLY_API_KEY
- powered_by : # true (default), false
+ application_id : "" # YOUR_APPLICATION_ID
+ index_name : "" # YOUR_INDEX_NAME
+ search_only_api_key : "" # YOUR_SEARCH_ONLY_API_KEY
+ powered_by : true # true (default), false
google:
- search_engine_id : # YOUR_SEARCH_ENGINE_ID
- instant_search : # false (default), true
+ search_engine_id : "" # YOUR_SEARCH_ENGINE_ID
+ instant_search : false # false (default), true
# SEO Related
-google_site_verification :
-bing_site_verification :
-yandex_site_verification :
-naver_site_verification :
+google_site_verification : ""
+bing_site_verification : ""
+yandex_site_verification : ""
+naver_site_verification : ""
# Social Sharing
twitter:
- username :
+ username : "rockthejvm"
+ icon : "icons/twitter.svg"
facebook:
- username :
- app_id :
- publisher :
-og_image : # Open Graph/Twitter default site image
+ username : "rockthejvm"
+ icon : "icons/facebook-square.svg"
+ app_id : ""
+ publisher : ""
+linkedin:
+ icon : "icons/linkedin.svg"
+og_image : "images/blog cover large.jpg"
# For specifying social profiles
# - https://developers.google.com/structured-data/customize/social-profiles
social:
- type : # Person or Organization (defaults to Person)
- name : # If the user or organization name differs from the site's name
- links: # An array of links to social media profiles
+ type : "Organization" # Person or Organization (defaults to Person)
+ name : "Rock the JVM" # If the user or organization name differs from the site's name
+ links:
+ - "https://twitter.com/rockthejvm"
+ - "https://facebook.com/rockthejvm"
+ - "https://github.com/rockthejvm"
+ - "https://youtube.com/rockthejvm"
+
+global_icons:
+ tag_icon : "icons/tag.svg"
+ calendar_icon : "icons/calendar.svg"
+ email_icon : "icons/email.svg"
+ read_time_icon : "icons/clock.svg"
+ feed_icon : "icons/rss.svg"
+ file_icon : "icons/file-alt.svg"
+ location_icon : "icons/location.svg"
+ link_icon : "icons/link.svg"
+ github_icon : "icons/github.svg"
+ youtube_icon : "icons/youtube.svg"
+ folder_open_icon : "icons/folder_open.svg"
+ no_icon : "icons/no_icon.svg"
# Analytics
analytics:
provider : false # false (default), "google", "google-universal", "google-gtag", "custom"
google:
- tracking_id :
- anonymize_ip : # true, false (default)
-
+ tracking_id : ""
+ anonymize_ip : false # true, false (default)
# Site Author
author:
- name : "Your Name"
- avatar : # path of avatar image, e.g. "/assets/images/bio-photo.jpg"
- bio : "I am an **amazing** person."
- location : "Somewhere"
- email :
+ name : "Daniel Ciocîrlan"
+ avatar : "https://res.cloudinary.com/dkoypjlgr/image/upload/c_auto,g_auto,h_110,w_110/v1715951687/daniel_mugshot_mktrla.jpg"
+ bio : "I'm a software engineer and the founder of Rock the JVM. I teach Scala, Kotlin, Java, big data and a variety of tech both live and in online courses."
+ location : "Bucharest, Romania"
+ email : "daniel@rockthejvm.com"
links:
- - label: "Email"
- icon: "fas fa-fw fa-envelope-square"
- # url: "mailto:your.name@email.com"
- label: "Website"
icon: "fas fa-fw fa-link"
- # url: "https://your-website.com"
+ url: "https://rockthejvm.com"
+ iconFile: "icons/link.svg"
- label: "Twitter"
icon: "fab fa-fw fa-twitter-square"
- # url: "https://twitter.com/"
+ url: "https://twitter.com/rockthejvm"
+ iconFile: "icons/twitter.svg"
- label: "Facebook"
icon: "fab fa-fw fa-facebook-square"
- # url: "https://facebook.com/"
+ url: "https://facebook.com/rockthejvm"
+ iconFile: "icons/facebook-square.svg"
- label: "GitHub"
icon: "fab fa-fw fa-github"
- # url: "https://github.com/"
- - label: "Instagram"
- icon: "fab fa-fw fa-instagram"
- # url: "https://instagram.com/"
+ url: "https://github.com/rockthejvm"
+ iconFile: "icons/github.svg"
+ - label: "YouTube"
+ icon: "fab fa-fw fa-youtube"
+ url: "https://youtube.com/rockthejvm"
+ iconFile: "icons/youtube.svg"
# Site Footer
footer:
links:
- label: "Twitter"
icon: "fab fa-fw fa-twitter-square"
- # url:
+ url: "https://twitter.com/rockthejvm"
+ iconFile: "icons/twitter.svg"
- label: "Facebook"
icon: "fab fa-fw fa-facebook-square"
- # url:
+ url: "https://facebook.com/rockthejvm"
+ iconFile: "icons/facebook-square.svg"
- label: "GitHub"
icon: "fab fa-fw fa-github"
- # url:
- - label: "GitLab"
- icon: "fab fa-fw fa-gitlab"
- # url:
- - label: "Bitbucket"
- icon: "fab fa-fw fa-bitbucket"
- # url:
- - label: "Instagram"
- icon: "fab fa-fw fa-instagram"
- # url:
-
+ url: "https://github.com/rockthejvm"
+ iconFile: "icons/github.svg"
+ - label: "YouTube"
+ icon: "fab fa-fw fa-youtube"
+ url: "https://youtube.com/rockthejvm"
+ iconFile: "icons/youtube.svg"
# Reading Files
include:
@@ -178,15 +197,13 @@ keep_files:
encoding: "utf-8"
markdown_ext: "markdown,mkdown,mkdn,mkd,md"
-
# Conversion
markdown: kramdown
-highlighter: rouge
+#highlighter: rouge
lsi: false
excerpt_separator: "\n\n"
incremental: false
-
# Markdown Processing
kramdown:
input: GFM
@@ -198,19 +215,16 @@ kramdown:
smart_quotes: lsquo,rsquo,ldquo,rdquo
enable_coderay: false
-
# Sass/SCSS
sass:
sass_dir: _sass
style: compressed # https://sass-lang.com/documentation/file.SASS_REFERENCE.html#output_style
-
# Outputting
permalink: /:categories/:title/
-paginate: 5 # amount of posts to show
+paginate: 10 # amount of posts to show
paginate_path: /page:num/
-timezone: # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-
+timezone: "" # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
# Plugins (previously gems:)
plugins:
@@ -219,6 +233,8 @@ plugins:
- jekyll-gist
- jekyll-feed
- jekyll-include-cache
+ - jekyll-archives
+ - jekyll-seo-tag
# mimic GitHub Pages with --safe
whitelist:
@@ -227,7 +243,7 @@ whitelist:
- jekyll-gist
- jekyll-feed
- jekyll-include-cache
-
+ - jekyll-archives
# Archives
# Type
@@ -246,17 +262,16 @@ tag_archive:
type: liquid
path: /tags/
# https://github.com/jekyll/jekyll-archives
-# jekyll-archives:
-# enabled:
-# - categories
-# - tags
-# layouts:
-# category: archive-taxonomy
-# tag: archive-taxonomy
-# permalinks:
-# category: /categories/:name/
-# tag: /tags/:name/
-
+jekyll-archives:
+ enabled:
+ - categories
+ - tags
+ layouts:
+ category: archive-taxonomy
+ tag: archive-taxonomy
+ permalinks:
+ category: /categories/:name/
+ tag: /tags/:name/
# HTML Compression
# - https://jch.penibelst.de/
@@ -265,7 +280,6 @@ compress_html:
ignore:
envs: development
-
# Defaults
defaults:
# _posts
diff --git a/_data/navigation.yml b/_data/navigation.yml
index 6f30866f3bed..07d0d1733c5d 100644
--- a/_data/navigation.yml
+++ b/_data/navigation.yml
@@ -1,7 +1,7 @@
# main links
main:
- - title: "Quick-Start Guide"
- url: https://mmistakes.github.io/minimal-mistakes/docs/quick-start-guide/
+ #- title: "Quick-Start Guide"
+ # url: https://mmistakes.github.io/minimal-mistakes/docs/quick-start-guide/
# - title: "About"
# url: https://mmistakes.github.io/minimal-mistakes/about/
# - title: "Sample Posts"
@@ -9,4 +9,4 @@ main:
# - title: "Sample Collections"
# url: /collection-archive/
# - title: "Sitemap"
- # url: /sitemap/
\ No newline at end of file
+ # url: /sitemap/
diff --git a/_drafts/2021-09-14-scalac.md b/_drafts/2021-09-14-scalac.md
new file mode 100644
index 000000000000..aa5456a440c6
--- /dev/null
+++ b/_drafts/2021-09-14-scalac.md
@@ -0,0 +1,152 @@
+---
+title: "Scala isn't Hard: How to Master Scala Step by Step"
+date: 2021-09-14
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "Scalac article lol."
+---
+
+## Intro
+
+With Scala 3 out in the wild for a few months, we're now in a new era of Scala development. In this article, we'll explore why people still consider Scala to be a hard language to learn, then we'll debunk that myth. I'll show you, step by step, how to master Scala from scratch, using the exact sequencing that I used for myself when I learned it. This is also how I personally train professional software engineers — and I've had the privilege of holding sessions at Adobe, Apple, Orange and other top companies — with great success over sessions of 2 or (sometimes) 3 days.
+
+You heard that right — you can master Scala in a weekend if you're committed and follow the steps I show you below.
+
+## Why Does Scala Seem to Be Hard?
+
+The fact of the matter is, no programming language is trivial. Every single one has its exciting features and frustrating bits. Scala is no different. But like any other programming language, learning Scala can be done quickly.
+
+From my experience teaching 41000+ people online at [Rock the JVM](https://rockthejvm.com), one common frustration that people have over (learning) Scala is that Scala has too many features and it's hard for people to connect them all. I can certainly agree: _if_ you learn chaotically and don't have a structure and sequencing already done, it's extremely hard to internalize anything. Type classes? Implicits? Givens? Higher-kinds? Of course it's hard when you have 3 million features in front of you and don't know which ones to pick first. Another frustration is the "getting stated" part and getting something to "just work". This is what makes many people let go of Scala quite early in the process.
+
+So that's exactly what I want to help with. In the rest of this article, I'm going to show you exactly what to focus on, and in what order, to learn Scala by yourself.
+
+As a prerequisite, you should be familiar with programming in general — with at least some other language — and have your computer science basics solid. If so, you can spend 1-2 hours on each of these sections in order, and I promise you'll be ahead of most people learning Scala.
+
+## 1 - Absolute Essentials
+
+If you're onboard for this journey, let's warm you up. This is where most people drop Scala because it's either 1) too frustrating because they have gaps in computer science basics, or 2) too boring because most tutorials spend too much time on these topics.
+
+The absolute basics of the Scala language include a few key ideas.
+
+- Defining values (`val`s) and understanding that values are the equivalent of constants in other languages.
+- Getting familiar with types in Scala: this is also where you get accustomed to the compiler and to _type inference_, the ability of the compiler to figure out the type of your values depending on what you write on the right hand side of an assignment.
+- Defining functions and recursion. This is a key concept: you must learn to think repetition in terms of _recursion_ instead of loops. This is one of the hardest concepts to internalize, for a beginner.
+- Learning a few non-essential, but fun tricks in Scala, like String interpolation.
+
+## 2 - Object-Oriented Programming
+
+Most of you know this already, but it's worth stressing again: Scala is not just for functional programming. In fact, the blend of FP _and OOP_ is what makes Scala so appealing in practice. After understanding the basics, this is your next stop. You should learn:
+
+- Classes, instances, fields and methods: this is your opportunity to make connections between Scala and your current experience (Java, Python, C++, C#, etc).
+- Method notation and special methods e.g. `apply`: this is one aspect that makes Scala extremely expressive. It was certainly the most fun thing to learn Scala when I started. The fact that you can say `mary likes "The Godfather"` instead of `mary.likes("The Godfather")` was music to my ears. I'm not going to give too many spoilers here.
+- Traits, abstract classes, inheritance and anonymous classes: another great opportunity to map existing concepts from other programming languages — e.g. interfaces and similar structures in Java.
+- Enums: one simple, but much needed feature that was added to Scala 3 and brings Scala to the same level with the most popular programming languages.
+- Case classes: a simple concept with powerful ramifications, because we use lightweight data structures in our code every day. So powerful that other languages e.g. Java and Kotlin have replicated this feature in the form of records and data classes respectively.
+- Exceptions: a similar mechanism for error signaling and handling, because of course Scala is based on the JVM.
+- Generics (just basics for now): the ability to reuse the same code on potentially unrelated types (e.g. collections). Scala takes this Java concept a few million miles further, but stop at the fundamentals for now.
+
+## 3 - Functional Programming Principles
+
+This is where most people dive in too early, but in my experience, this is not the right first step. The previous two sections serve as foundations for what you'll discover here.
+
+Why?
+
+Because Scala, being based on the JVM — which was built for Java, one of the canonical OO languages — has to resort to certain tricks to achieve some of the key aspects of functional programming: the ability to pass functions as arguments and to return them as results. In other words, the ability to handle functions as "first class" elements of the language, built for a platform that only supports OOP, needs some OO foundations.
+
+- Now you're ready to understand what a function value is in Scala. You can get familiar with the `FunctionN` family of traits in Scala. With this knowledge, I personally felt like a new section of my brain was unlocked, when I learned this.
+- Learn lambdas, i.e. anonymous functions. With the idea above, this concept will now feel trivial.
+- Now that you can pass functions as arguments and use them like any other values, learn the concept of _higher-order functions_. Play with HOFs like map, flatMap, filter and learn the essential bits of the _collection library_.
+- Learn `for` comprehensions. If you know map, flatMap and filter, you'll understand this syntax sugar easily. Invent your own data structures — or replicate existing ones — and use `for` comprehensions on them.
+- Learn a few useful data structures like Option and Try. They will prepare you for further abstractions down the line.
+
+## 4 - Pattern Matching
+
+The previous sections gave you the absolute basics to work with Scala on a daily basis. From now on, you're ready to dive into either more abstract/complex topics, or explore more Scala-specific features.
+
+Pattern matching is one Scala-specific feature that has been around for more than a decade, and remains far ahead of the attempts of other languages _cough_ Java _cough_ to replicate it.
+
+Learn what a pattern matching expression is and why it's useful. Deconstruct case classes and constants first, that will cover 90% of your use case in practice.
+
+After that, if you have some spare time and you're ready to show off to your Scala teammates, learn some [nifty tricks](https://blog.rockthejvm.com/8-pm-tricks/) with pattern matching.
+
+At this point, you can be a productive Scala developer for simple to medium projects. The topics above are what I teach in my [Scala 3 Essentials](https://rockthejvm.com/p/scala) course on Rock the JVM, and you can also follow the above steps on your own to learn by yourself in just a day of focused work.
+
+## 5 - Advanced Functional Programming
+
+After getting the foundations straight, you should now be ready to dive into more abstract and complex topics. Some advanced functional programming concepts are in order:
+
+- Partial functions
+- Partially applied functions and a compiler process called eta-expansion
+- Lazy evaluation
+
+With these Scala features, you will unlock some new mental models around programming concepts you've been using all the time. For example, I regularly teach people to think of collections as functions. This usually blows their mind as much as it did mine when I discovered it. Many Scala collections, considered mathematically are (total or partial) functions mapping domains. This will prepare you for further abstractions down the line.
+
+## 6 - Functional Asynchronous Programming
+
+We haven't discussed threads so far, and for good reason. At this point, you are ready to explore the JVM threading model and the kind of problems that we encounter on a daily basis:
+
+- creating threads
+- creating and managing thread pools, submitting tasks
+- race conditions and synchronization
+- deadlocks and livelocks
+
+Some of you Java gurus might want to jump over these, as they are exactly the same for Java. However, with this JVM threading foundations, you can now explore Futures in Scala: a functional-programming-ready primitive that allows you to write parallel and concurrent code while keeping the functional programming principles intact.
+
+## 7 - Contextual Abstractions
+
+In this section, we can dive into more abstract territory. Here's what to study:
+
+- `given` instances and `using` clauses. These might come across as a bit unintuitive at first, because you might be thinking, "why on earth would I need this?". Trust me, this unlocks some crazy new potential. If you're learning Scala 2, search for `implicit val` and implicit arguments — don't search for other `implicit` stuff.
+- Extension methods. This is the capability of Scala to add new methods to existing types, even after they're defined. If you're an OOP guru and you're about to shout that this destroys final classes — don't stress about it. This has nothing to do with OOP.
+- The above features are called "contextual abstractions". There are more of them, but these are the most important. Learn how to organize them, i.e. how to place them in packages and objects and how to import them effectively. The compiler follows a certain sequence of steps so your code will compile with them. Understand these phases.
+- Learn what a _type class_ is. Build one (or more) out of givens and extension methods. An exercise that I often use is a mini-JSON-serialization library.
+- Optionally, learn context functions and implicit conversions. You might not need them.
+
+## 8 - Master the Type System
+
+Scala's type system is almost unparalleled. The language has a million features that allows us to express some extremely general and powerful code — but this, I suspect, is one of the reasons why some people stay away from Scala as well. It doesn't need to be hard. You need the right structure. Here it is:
+
+- Learn something called _trait linearization_ in Scala. Because of multiple inheritance, trait hierarchies can get into trouble. Learn how Scala solves that.
+- Go one level deeper with generics, and learn about _variance_. This feature is widely used in ZIO, one of the most popular libraries for Scala.
+- Learn about _variance positions_ and how to overcome some cryptic compiler errors. You'll need this, otherwise you'll spend (like I did) hours banging your head against the wall.
+- Learn about some new types added in Scala 3: literal types, union types and intersection types. These have some interesting properties and will help unify your code and/or your APIs.
+- Learn about _self-types_. You'll sometimes notice something like `class A { self: B => ` and wonder if that's a lambda. It's not. It looks weird, but it's quite useful sometimes. Learn the difference between the type restrictions enforced by self-types and regular OO inheritance.
+- Finally, learn about _higher-kinded types_ — the ability to define generic types with type arguments that are _themselves_ generic. This opens the door to some really cool functionality. Coupled with type classes, this feature is at the core of Cats, the popular library for Scala functional programming.
+
+These other four sections are what I teach in my [Advanced Scala 3](https://rockthejvm.com/p/advanced-scala) course, plus a few other bits that are not really necessary if you want to learn Scala really fast. As before: if you follow the above topics in order, you'll be ahead of 95% of people learning Scala.
+
+## 9 - Understand the Scala Ecosystem
+
+If you really want to be a productive Scala developer and want to stay — and most new Scala devs don't want to go back, I know for a fact — the next step I recommend you take is to familiarize yourself with libraries and tools based on Scala. There are quite a few of them and you'll be able to get pretty much anything done with what the ecosystem can currently offer. Here are some of the most popular:
+
+- [Cats Effect](https://typelevel.org/cats-effect) for the foundational layer of concurrent systems, based on pure functional programming. Has an amazingly fast runtime and powerful abstractions that will allow you to write concurrent code with ease. I also teach this at Rock the JVM.
+- [Cats](https://typelevel.org/cats) for functional programming abstractions. It's a little more mathematical — it's a playful name for _categories_ — but they are all rooted in practice and in the need to organize concepts around your codebase.
+- [ZIO](https://zio.dev) is another mini-ecosystem of tools for the foundation of concurrent systems. It's a competitor for Cats Effect, with a runtime just as fast, built on similar principles, but with a different philosophy in terms of organization and APIs.
+- [Akka](https://akka.io) is a complete set of tools not just for concurrent systems, but for _distributed_ applications, which is super-powerful. It's built on the actor model, which is what OOP was initially dreamed to be, with amazing results in real critical systems. I've created an entire series on Akka on Rock the JVM.
+- Data streaming systems like [Apache Kafka](https://kafka.apache.org) are rocking the distributed systems world.
+- Big data libraries like [Apache Spark](https://spark.apache.org) have become the de-facto big data processing tools used everywhere.
+- [Play](https://www.playframework.com/) is a complete framework for building web applications, with Akka running under the hood.
+- [Shapeless](https://github.com/milessabin/shapeless) pushes the boundaries of what's possible in the Scala language itself.
+
+My advice after learning Scala: dip your toes in at least one of them that looks interesting.
+
+## 10 - Practice Doing Something You Would Use
+
+Finally, I recommend _building something_. Don't let that new knowledge go to waste. Even if you use just the plain Scala language with no other libraries, you should make something out of nothing.
+
+What should you do?
+
+My recommendation has always been: build something that you would use yourself. A web scraper, a file synchronizer, a Twitter bot, a chess engine, generative art, fractals, an ML pipeline, the number pi with 10 billion decimals, whatever looks interesting. Satisfy your curiosity with the new tool that you've discovered.
+
+If no ideas come to mind, try contributing to the above tools. They're all open-source and extra pairs of eyes and hands are always welcome.
+
+## Conclusion
+
+So why should you learn Scala at all? Here are a few reasons:
+
+- It's a rare skill and many companies are desperately searching for Scala developers. It has some of the best-paid programming positions.
+- It's a giant leap for your mindset as a developer. Even if you don't end up writing Scala (which is rare), you have some mental models that you can apply to any programming environment.
+- It's just plain fun. Scala can be so expressive, so compact and yet so powerful. With some of the tools above, you can write in 20 lines what other developers need 2000 lines.
+
+I hope the above ideas and tips were useful. Tens of thousands who learned from [Rock the JVM](https://rockthejvm.com) have followed them, with some amazing results for their projects and their careers.
diff --git a/_includes/analytics-providers/google-gtag.html b/_includes/analytics-providers/google-gtag.html
index 16d0cf176bac..1c319c5f67e9 100644
--- a/_includes/analytics-providers/google-gtag.html
+++ b/_includes/analytics-providers/google-gtag.html
@@ -1,6 +1,6 @@
-
-{{ page.last_modified_at | date: "%B %-d, %Y" }}
+
+
+ {% if site.global_icons.calendar_icon %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.calendar_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
+ {{ site.data.ui-text[site.locale].date_label | default: "Updated:" }}
+
+ {{ page.last_modified_at | date: "%B %-d, %Y" }}
+
{% elsif page.date %}
- {{ page.date | date: "%B %-d, %Y" }}
+
+
+ {% if site.global_icons.calendar_icon %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.calendar_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
+ {{ site.data.ui-text[site.locale].date_label | default: "Updated:" }}
+
+ {{ page.date | date: "%B %-d, %Y" }}
+
{% endif %}
diff --git a/_includes/page__hero.html b/_includes/page__hero.html
index 3f55aaa60ac5..9e6bf0223e9d 100644
--- a/_includes/page__hero.html
+++ b/_includes/page__hero.html
@@ -48,4 +48,23 @@
{% if page.header.caption %}
{{ page.header.caption | markdownify | remove: "" | remove: "
" }}
{% endif %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/_includes/page__meta.html b/_includes/page__meta.html
index 1afc3d8f85eb..d2c119d63091 100644
--- a/_includes/page__meta.html
+++ b/_includes/page__meta.html
@@ -1,10 +1,26 @@
{% assign document = post | default: page %}
+{% assign author = page.author | default: page.authors[0] | default: site.author %}
+{% assign author = site.data.authors[author] | default: author %}
{% if document.read_time or document.show_date %}
{% if document.show_date and document.date %}
{% assign date = document.date %}
-
+ {% capture svg %}
+ {% include {{ site.global_icons.calendar_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
{{ date | date: "%B %-d, %Y" }}
{% endif %}
@@ -16,7 +32,21 @@
{% assign words = document.content | strip_html | number_of_words %}
-
+ {% capture svg %}
+ {% include {{ site.global_icons.read_time_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
{% if words < words_per_minute %}
{{ site.data.ui-text[site.locale].less_than | default: "less than" }} 1 {{ site.data.ui-text[site.locale].minute_read | default: "minute read" }}
{% elsif words == words_per_minute %}
diff --git a/_includes/social-share.html b/_includes/social-share.html
index 0b377982b268..a681b1ab695f 100644
--- a/_includes/social-share.html
+++ b/_includes/social-share.html
@@ -3,9 +3,60 @@
{{ site.data.ui-text[site.locale].share_on_label | default: "Share on" }}
{% endif %}
-
+
- Facebook
+ {% if site.facebook.icon %}
+
+ {% capture svg %}
+ {% include {{ site.facebook.icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
+ Facebook
+
- LinkedIn
+ {% if site.linkedin.icon %}
+
+ {% capture svg %}
+ {% include {{ site.linkedin.icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
+ LinkedIn
+
diff --git a/_includes/subscribe.html b/_includes/subscribe.html
new file mode 100644
index 000000000000..8522a6f4578f
--- /dev/null
+++ b/_includes/subscribe.html
@@ -0,0 +1,19 @@
+
diff --git a/_includes/tag-list.html b/_includes/tag-list.html
index e0d02bfa561a..fe0f94261424 100644
--- a/_includes/tag-list.html
+++ b/_includes/tag-list.html
@@ -14,7 +14,24 @@
{% assign tag_hashes = page_tags | split: ',' | sort %}
-
+ {% if site.global_icons.tag_icon %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.tag_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% else %}
+
+ {% capture svg %}
+ {% include {{ site.global_icons.no_icon }} %}
+ {% endcapture %}
+ {{ svg | remove: '' }}
+
+ {% endif %}
+ {{ site.data.ui-text[site.locale].tags_label | default: "Tags:" }}
+
{% for hash in tag_hashes %}
{% assign keyValue = hash | split: '|' %}
diff --git a/_layouts/single.html b/_layouts/single.html
index 71cb4e83dc66..f744d978559c 100644
--- a/_layouts/single.html
+++ b/_layouts/single.html
@@ -2,6 +2,10 @@
layout: default
---
+{% assign author = page.author | default: page.authors[0] | default: site.author %}
+{% assign author = site.data.authors[author] | default: author %}
+
+
{% if page.header.overlay_color or page.header.overlay_image or page.header.image %}
{% include page__hero.html %}
{% elsif page.header.video.id and page.header.video.provider %}
@@ -35,7 +39,24 @@
{% if page.toc %}
@@ -44,6 +65,8 @@
{% if page.link %}{% endif %}
+ {% include subscribe.html %}
+
{% if site.data.ui-text[site.locale].meta_label %}
{{ site.data.ui-text[site.locale].meta_label }}
@@ -60,6 +83,8 @@ {{ site.data.ui-text[site.locale].meta_label }}
{% comment %}{% endcomment %}
diff --git a/docs/_pages/category-archive.md b/_pages/category-archive.md
similarity index 95%
rename from docs/_pages/category-archive.md
rename to _pages/category-archive.md
index 4cb3860e91be..6f458d5c9f65 100644
--- a/docs/_pages/category-archive.md
+++ b/_pages/category-archive.md
@@ -3,4 +3,4 @@ title: "Posts by Category"
layout: categories
permalink: /categories/
author_profile: true
----
+---
\ No newline at end of file
diff --git a/docs/_pages/tag-archive.md b/_pages/tag-archive.md
similarity index 95%
rename from docs/_pages/tag-archive.md
rename to _pages/tag-archive.md
index 3f4e3f0df86c..ec32274d83ce 100644
--- a/docs/_pages/tag-archive.md
+++ b/_pages/tag-archive.md
@@ -3,4 +3,4 @@ title: "Posts by Tag"
permalink: /tags/
layout: tags
author_profile: true
----
+---
\ No newline at end of file
diff --git a/test/_pages/year-archive.md b/_pages/year-archive.md
similarity index 67%
rename from test/_pages/year-archive.md
rename to _pages/year-archive.md
index 1021452ad951..7b25e752893d 100644
--- a/test/_pages/year-archive.md
+++ b/_pages/year-archive.md
@@ -1,6 +1,6 @@
---
title: "Posts by Year"
-permalink: /year-archive/
+permalink: /posts/
layout: posts
author_profile: true
----
+---
\ No newline at end of file
diff --git a/_posts/2020-03-29-8-pm-tricks.md b/_posts/2020-03-29-8-pm-tricks.md
new file mode 100644
index 000000000000..829b6e4f6f6b
--- /dev/null
+++ b/_posts/2020-03-29-8-pm-tricks.md
@@ -0,0 +1,219 @@
+---
+title: "8 Scala Pattern Matching Tricks"
+date: 2020-03-29
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala]
+excerpt: "Pattern matching is one of the most powerful Scala features. Learn to use it to the max so you can write your best Scala code."
+---
+You can't be a Scala programmer and say you've never used pattern matching.
+
+Pattern matching is one of the most powerful Scala features. It allows one to test lots of values and conditions without nested and chained if-else expressions.
+
+```scala
+val aNumber = 44
+val ordinal = aNumber match {
+ case 1 => "first"
+ case 2 => "second"
+ case 3 => "third"
+ case _ => aNumber + "th" // ignore the English grammar will you please
+}
+```
+
+However, you also probably know that pattern matching is more than a switch expression, but more like a switch on steroids:
+
+```scala
+case class Person(name: String, age: Int, favoriteMovies: List[String])
+val bob = Person("Bob", 34, List("Inception", "The Departed"))
+
+val describeBob = bob match {
+ case Person(n, a, movies) => s"$n is $a years old and likes ${movies.mkString(",")}"
+ case _ => "I don't know what you're talking about"
+}
+```
+
+In other words, pattern matching can deconstruct case class instances into their consistuent parts, which we can then reuse in the resulting expression on the right-hand side of =>.
+
+Let me give you some more powerful patterns that Scala supports out of the box, but you might not have known existed.
+
+## 1. List extractors
+
+Lists can be deconstructed with pattern matching in a number of powerful ways. Let me build a list:
+
+```scala
+val countingList = List(1,2,3,42)
+```
+
+You can extract any element out of this list with a pattern that looks like the case class constructor:
+
+```scala
+val mustHaveThree = countingList match {
+ case List(_, _, 3, somethingElse) => s"A-HA! I've got a list with 3 as third element, I found $somethingElse after"
+}
+```
+
+This pattern matches a list with exactly 4 elements, in which we don't care about the first two, the third one must be exactly 3, and the fourth can be anything, but we name it somethingElse so we can reuse it in the s-interpolated string.
+
+## 2. Haskell-like prepending
+
+If I consider the same list as before, I can extract the head and tail of the list as follows:
+
+```scala
+val startsWithOne = countingList match {
+ case 1 :: someOtherElements => "This lists starts with one, the rest is $someOtherElements"
+}
+```
+
+Don't ask how this is possible yet - that will be the subject of an upcoming advanced article. The prepend pattern is often very useful in code that processes a list, but when you don't know in advance whether the list is empty or not, so you write it like this:
+
+```scala
+def processList(numbers: List[Int]): String = numbers match {
+ case Nil => ""
+ case h :: t => h + " " + processList(t)
+}
+```
+
+This style of handling a list may be very familiar to those of you who know a bit of Haskell.
+
+## 3. List vararg pattern
+
+The first pattern we showed above can only constrain a list to a definitive number of elements. What if you don't know (or care about) the number of elements in advance?
+
+```scala
+val dontCareAboutTheRest = countingList match {
+ case List(_, 2, _*) => "I only care that this list has 2 as second element"
+}
+```
+
+The _* is the important bit, which means "any number of additional arguments". This pattern is much more flexible because an (almost) infinite number of lists can match this pattern, instead of the 4-element list pattern we had before. The only catch with _* is that it must be the last bit in the pattern. In other words, the case List(_, 2, _*, 55) will not compile, for example.
+
+## 4. Other list infix patterns
+
+It's very useful when we can test the head of the list, or even the elements inside the list. But what if we want to test the last element of the list?
+
+```scala
+val mustEndWithMeaningOfLife = countingList match {
+ case List(1,2,_) :+ 42 => "found the meaning of life!"
+}
+```
+
+The :+ is the append operator, which works much like :: from the point of view of pattern matching. You can also use the +: prepend operator (for symmetry), but we prefer ::. A nice benefit of the append operator is we can combine it with the vararg pattern for a really powerful structure:
+
+```scala
+val mustEndWithMeaningOfLife2 = countingList match {
+ case List(1, _*) :+ 42 => "I really don't care what comes before the meaning of life"
+}
+```
+
+Which overcomes some of the limitations of the vararg pattern above.
+
+## 5. Type specifiers
+
+Sometimes we really don't care about the values being matched, but only their type.
+
+```scala
+def gimmeAValue(): Any = { ... }
+
+val gimmeTheType = gimmeAValue() match {
+ case _: String => "I have a string"
+ case _: Int => "I have a number"
+ case _ => "I have something else"
+}
+```
+
+The :String bit is the important part. It allows the cases to match only those patterns which conform to that type. One very useful scenario where this is particularly useful is when we catch exceptions:
+
+```scala
+try {
+ ...
+} catch {
+ case _: IOException => "IO failed!"
+ case _: Exception => "We could have prevented that!"
+ case _: RuntimeException => "Something else crashed!"
+}
+```
+
+(spoiler: catching exceptions is also based on pattern matching!)
+
+The drawback with type guards is that they are based on reflection. Beware of performance hits!
+
+## 6. Name binding
+
+I've seen the following pattern more times than I can count:
+
+```scala
+def requestMoreInfo(p: Person): String = { ... }
+
+val bob = Person("Bob", 34, List("Inception", "The Departed"))
+
+val bobsInfo = bob match {
+ case Person(name, age, movies) => s"$name's info: ${requestMoreInfo(Person(name, age, movies))}"
+}
+```
+
+We deconstruct a case class only to re-instantiate it with the same data for later. If we didn't care about any field in the case class, that would be fine, because we would use a type specifier (see above). Even that is not 100% fine because we rely on reflection. But what if we care about some fields (not all) and the entire instance, so we can reuse those?
+
+```scala
+val bobsInfo = bob match {
+ case p @ Person(name, _, _) => s"$name's info: ${requestMoreInfo(p)}"
+}
+```
+
+Answer: name the pattern you're matching (see the p @) so you can reuse it later. You can even name sub-patterns:
+
+```scala
+val bobsInception = bob match {
+ case Person(name, _, movies @ List("Inception", _*)) => s"$name REALLY likes Inception, some other movies too: $movies"
+}
+```
+
+## 7. Conditional guards
+
+If you're like me, you probably tried at least once to pattern match something that satisfies a condition, and because you only knew the "anything" and "constant" patterns, you gave up pattern matching and used chained if-elses instead.
+
+```scala
+val ordinal = gimmeANumber() match {
+ case 1 => "first"
+ case 2 => "second"
+ case 3 => "third"
+ case n if n % 10 == 1 => n + "st"
+ case n if n % 10 == 2 => n + "nd"
+ case n if n % 10 == 3 => n + "rd"
+ case n => n + "th"
+}
+```
+
+As you can see above, the if guards are there directly in the pattern. Also notice that the condition does not have parentheses.
+
+## 8. Alternative patterns
+
+For the situations where you return the same expression for multiple patterns, you don't need to copy and paste the same code.
+
+```scala
+val myOptimalList = numbers match {
+ case List(1, _, _) => "I like this list"
+ case List(43, _*) => "I like this list"
+ case _ => "I don't like it"
+}
+```
+
+You can combine the patterns where you return the same expression into a single pattern:
+
+```scala
+val myOptimalList = numbers match {
+ case List(1, _, _) | List (43, _*) => "I like this list"
+ case _ => "I don't like it"
+}
+```
+
+The only drawback of this pattern is that you can't bind any names, because the compiler can't ensure those values are available on the right-hand side.
+
+This pattern is useful in practice for a lot of scenarios, for example when you want to handle many kinds of exceptions:
+
+```scala
+try {
+ ...
+} catch {
+ case _: RuntimeException | _: IOException => ""
+}
+```
diff --git a/_posts/2020-03-31-contravariance.md b/_posts/2020-03-31-contravariance.md
new file mode 100644
index 000000000000..32bdb9d404a7
--- /dev/null
+++ b/_posts/2020-03-31-contravariance.md
@@ -0,0 +1,89 @@
+---
+title: "Why is Scala contravariance so hard?"
+date: 2020-03-31
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "The Scala type system is powerful, but also hard. We look at contravariance in Scala and try to make some sense out of it."
+---
+
+> We discuss variance and variance positions in depth in the [Advanced Scala](https://rockthejvm.com/p/advanced/scala) course. Check it out!
+
+This article is for the Scala programmer who's either getting started with Scala generics, OR who has been using generics in their basic form, i.e. just attach type arguments, like
+
+```scala
+val list: List[Int] = List(1,2,3)
+```
+
+## So what's variance?
+
+It's that cute little question: if dogs are animals, could lists of dogs be considered lists of animals as well? This is the variance question, i.e. whether the subtype relationship can be transferred to generic types.
+
+If the answer to the variance question is "yes", then we consider the generic type covariant, and in Scala we write a + next to the type argument, like
+
+```scala
+abstract class List[+T]
+```
+
+which then allows us to operate with lists polymorphically:
+
+```scala
+val laika: Animal = new Dog("Laika")
+val myDogs: List[Animal] = List(lassie, hachi, laika)
+```
+
+This is more easily understood. Dog subtype of Animal, therefore List[Dog] subtype of List[Animal].
+
+However, that's not the only possible answer. We can also have no variance (no + sign), which makes a generic type invariant. This is the Java way of dealing with generics. Dog subtype of Animal? I don't care. List[Dog] and List[Animal] are two different types, with no relationship between them. That means if you write
+
+```scala
+val myDogs: List[Animal] = List(lassie, hachi, laika)
+```
+
+the code will not compile, because the compiler expects a List[Animal] and you're giving it a List[Dog].
+
+That's not the end of it, though. There is still one more possible answer to the yes/no variance question. We dealt with the yes and no answers, but there is a third one, which sounds like "hell no" or "no, quite the opposite". This is contravariance.
+
+## Enter contravariance
+
+Let's get this straight. So we have Dog subtype of Animal, and we're wondering what could be the relationship between List[Dog] and List[Animal]. In the contravariance answer, we would have a list of animals being a subtype of a list of dogs, which is the exact opposite of covariance above. The following will be very confusing.
+
+```scala
+class MyList[-T]
+val myAnimals: MyList[Dog] = MyList(crocodile, kitty, lassie) // some animals
+```
+
+When you write a minus in the generic type, that's a marker to the compiler that the generic type will have the subtype relationship exactly opposite to the types it wraps. Namely, MyList[Animal] is a subtype of a MyList[Dog]. The code above would compile, but we would not be happy, because it makes no sense. Why would we write that for a list? Why would a list of animals be a SUBTYPE of a list of dogs?
+
+## The why
+
+Why do we need contravariance? When should we use it?
+
+## The because
+
+Many Scala developers dismiss the variance concepts (contravariance in particular) as obscure and purely academic. However, if we think about other scenarios of real life, we can actually find some meaning in variance. Let's go back to the Dog-Animal relationship and let's try to imagine something like a Vet, which can heal an animal.
+
+```scala
+trait Vet[-T] { // we can also insert an optional -T <: Animal here if we wanted to impose a type constraint
+ def heal(animal: T): Boolean
+}
+```
+
+I've already defined it with a -T, for the following reason: if you ask me "Daniel, gimme a vet for my dog" and I'll give you a vet which can heal ANY animal, not just your dog, your dog will live.
+
+```scala
+val myDog = new Dog("Buddy")
+val myVet: Vet[Dog] = new Vet[Animal] { ... }
+myVet.heal(myDog)
+```
+
+Look at the above code until it makes sense. We're declaring a Vet[Dog], and instead we have a Vet[Animal], with the meaning that the vet can heal any animal, therefore it can work on my dog as well. The code will compile, our buddy will live, and we would be happy.
+
+## The punchline
+
+So when is it best to use covariance and contravariance? Clearly a contravariant list doesn't make sense (as we saw at the beginning), and in the exact same style, a covariant vet doesn't make sense.
+
+Here's a rule of thumb: when your generic type "contains" or "produces" elements of type T, it should be covariant. When your generic type "acts on" or "consumes" elements of type T, it should be contravariant.
+
+Examples of covariant concepts: a cage (holds animals), a garage (holds cars), a factory (creates objects), a list (and any other collection).
+Examples of contravariant concepts: a vet (heals animals), a mechanic (fixes cars), a garbage pit (consumes objects), a function (it acts on/it's applied on arguments).
diff --git a/_posts/2020-04-01-a-5-minute-akka-http-client.md b/_posts/2020-04-01-a-5-minute-akka-http-client.md
new file mode 100644
index 000000000000..a64746f82431
--- /dev/null
+++ b/_posts/2020-04-01-a-5-minute-akka-http-client.md
@@ -0,0 +1,113 @@
+---
+title: "Sending HTTP Requests in Scala and Akka in 5 minutes"
+date: 2020-04-01
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka, akka http, how to]
+excerpt: "Learn to use Akka HTTP with Scala and send HTTP requests in just a few minutes with the Akka HTTP server DSL."
+---
+This article is for the Scala programmer who wants to run one-off HTTP requests quickly. The thinking style assumed is "I don't want to care too much, I'll give you a payload, you just give me a future containing your response". With minimal boilerplate, we'll do exactly that with Akka HTTP in 5 minutes.
+
+The Rock the JVM blog is built with me typing my posts in plain text with minimal Markdown formatting, and then generating a uniform HTML out of it, with a simple Scala parser (I hate typing HTML). For syntax highlighting, I use markup.su/highlighter, which happens to have a REST endpoint. Naturally, I don't want to do it by hand, so as my HTML is generated, the syntax is automatically retrieved via Akka HTTP as client, with little code. My HMTL generator currently has less than 100 lines of code in total.
+
+In this article, I'm going to get you started with the simplest Akka HTTP client API in 5 minutes.
+
+## The tiny setup
+
+First, you need to add the Akka libraries. Create an SBT project in your dev environment (I recommend IntelliJ), then add this to the build.sbt file:
+(if you've never used SBT before, the build.sbt file describes all the libraries that the project needs, which the IDE will download automatically)
+
+```scala
+val akkaVersion = "2.5.26"
+val akkaHttpVersion = "10.1.11"
+
+libraryDependencies ++= Seq(
+ // akka streams
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion,
+ // akka http
+ "com.typesafe.akka" %% "akka-http" % akkaHttpVersion,
+ "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion,
+)
+```
+
+Then in a Scala application I'm going to write a piece of small boilerplate, because Akka HTTP needs an actor system to run:
+
+```scala
+implicit val system = ActorSystem()
+implicit val materializer = ActorMaterializer()
+import system.dispatcher
+```
+
+## Sending HTTP requests
+
+Now we'll need to start sending HTTP requests. I'll use the exact HTTP API I'm using for the blog: http://markup.su/highlighter/api. The API says it needs GET or POST requests to /api/highlighter, with the parameters "language", "theme" and "source" in a request with content type application/x-www-form-urlencoded.
+
+So let me create a piece of Scala code:
+
+```scala
+val source =
+"""
+ |object SimpleApp {
+ | val aField = 2
+ |
+ | def aMethod(x: Int) = x + 1
+ |
+ | def main(args: Array[String]) = {
+ | println(aMethod(aField))
+ | }
+ |}
+""".stripMargin
+```
+
+and then let me create an HTTP request for it:
+
+```scala
+ val request = HttpRequest(
+ method = HttpMethods.POST,
+ uri = "http://markup.su/api/highlighter",
+ entity = HttpEntity(
+ ContentTypes.`application/x-www-form-urlencoded`,
+ s"source=${URLEncoder.encode(source.trim, "UTF-8")}&language=Scala&theme=Sunburst"
+ )
+ )
+```
+
+where I've named the arguments in the call for easy reading. In Akka HTTP, an HttpRequest contains the HTTP method (POST in our case), the URI and a payload in the form of an HttpEntity. We specify the content type per the description specified in the API - notice the backticks for the name of the field - and the actual string we want to send, as described by the API. In practice, you can send other strings, like JSONs - I'll show you how to auto-convert your data types to JSON auto-magically in another article.
+
+Then, we actually need to send our request:
+
+```scala
+ def simpleRequest() = {
+ val responseFuture = Http().singleRequest(request)
+ responseFuture.flatMap(_.entity.toStrict(2 seconds)).map(_.data.utf8String).foreach(println)
+ }
+```
+
+The Akka HTTP client call is simple: just call the singleRequest method. You obtain a Future containing an HTTP response, which we can then unpack. We use its entity (= its payload) and convert it to a strict entity, meaning take its whole content in memory. We then take its data which is a sequence of bytes, and convert that to a string. And we're done.
+
+## Hide it all
+
+We can create a very nice method which hides this all away:
+
+```scala
+ def highlightCode(myCode: String): Future[String] = {
+ val responseFuture = Http().singleRequest(
+ HttpRequest(
+ method = HttpMethods.POST,
+ uri = "http://markup.su/api/highlighter",
+ entity = HttpEntity(
+ ContentTypes.`application/x-www-form-urlencoded`,
+ s"source=${URLEncoder.encode(myCode.trim, "UTF-8")}&language=Scala&theme=Sunburst"
+ )
+ )
+ )
+
+ responseFuture
+ .flatMap(_.entity.toStrict(2 seconds))
+ .map(_.data.utf8String)
+ }
+```
+
+And then you can go on with your day: pass a string, expect a future containing an HTML highlighting. All done!
+
+If you want to practice sending HTTP requests as an exercise, you can use https://jsonplaceholder.typicode.com/ for dummy APIs, the same principle applies.
diff --git a/_posts/2020-04-03-self-types.md b/_posts/2020-04-03-self-types.md
new file mode 100644
index 000000000000..47ef4fe77ac4
--- /dev/null
+++ b/_posts/2020-04-03-self-types.md
@@ -0,0 +1,102 @@
+---
+title: "Self-types in Scala, Real Quick"
+date: 2020-04-03
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "Self types are a really interesting way to enforce type constraints in Scala. Learn to use it in a few minutes."
+---
+In this article I'm going to address self-types, which are a pretty strange piece of Scala syntax and functionality. Many Scala devs go on writing code for months without any encounter with self-types, then they get baffled by the structure when they need to dig into some library code.
+
+I'm going to address two different wildly different angles to self-types:
+ 1) What the weird structure means
+ 2) What do you do when you want to enforce that a trait MUST be mixed into a type you're defining.
+
+I'll start with #2 and get back to #1 into a (hopefully) logical progression.
+
+## Enforcing type constraints
+
+Let's say you have two separate type hierarchies:
+
+```scala
+// person hierarchy
+trait Person {
+ def hasAllergiesFrom(thing: Edible): Boolean
+}
+trait Child extends Person
+trait Adult extends Person
+
+// diet hierarchy
+trait Diet {
+ def eat(thing: Edible): Boolean
+}
+trait Carnivore extends Diet
+trait Vegetarian extends Diet
+```
+
+The problem is that you want the diet to be applicable to Persons only. Not only that, but your functionality actually relies on logic from the Person class. For example, you need a person's age or weight while you implement your Diet API. This is often an issue when you design library APIs. There are various options to do it.
+
+## Option 1: Inheritance
+
+```scala
+// option 1
+trait Diet extends Person
+```
+
+This option makes all the (non-private) Person functionality available to Diet. However, this obviously makes a mess out of two otherwise clean and separate concept hierachies.
+
+## Option 2: Generics
+
+```scala
+// option 2
+trait Diet[P <: Person]
+```
+
+This option adds a degree of separation by adding a type argument. A bit better, but with its own problems. You have access to the Person type, but not to its methods - you would need an instance of Person to access the logic, but then you'd need to pass a Person as a constructor argument. Until Scala 3 comes along, there's no way for you to do that, or enforce any Diet subclasses to pass a person as an argument. Not to mention variance problems. Is a vegetarian diet for an adult also applicable to teenagers? Which implementations can I reuse for which types?
+
+So here's an option that can make it clean:
+
+## Option 3: Self-types
+
+```scala
+// option 3
+trait Diet { self: Person =>
+
+}
+```
+
+This is called a self-type. It looks like a lambda structure, but it has a completely different meaning. When you see something like this in a class or trait definition, you need to read it as "whichever class implements the Diet trait MUST ALSO implement the Person trait". For example:
+
+```scala
+class VegAthlete extends Vegetarian with Adult
+```
+
+This class implements both the Diet trait (via Vegetarian) and the Person trait (via Adult). The main advantage of using this structure is that you can use the Person functionality directly in the Diet trait, without otherwise creating any type relationship between the two hierarchies:
+
+```scala
+trait Diet { self: Person =>
+ def eat(thing: Edible): Boolean =
+ if (self.hasAllergiesFrom(thing)) false
+ else ...
+}
+```
+
+Since you created Diet as a self-type, you already assume that whoever implements Diet will also implement Person, so you will have access to the Person methods. So you can directly use them ahead of time, in the Diet trait.
+
+## The Punchline
+
+People constantly ask me what the fundamental difference between self-types and inheritance is, aside from the scenario above. "Why can't we solve this with plain inheritance?". The fundamental difference is subtle. Here's the quick "remember this" lesson:
+
+```scala
+class Dog extends Animal
+```
+
+This means Dog IS AN Animal. When you say
+
+```scala
+trait Diet { self: Person =>
+
+}
+```
+
+you say that Diet REQUIRES A Person. So remember the "is a" vs "requires a" distinction.
diff --git a/_posts/2020-04-04-controllable-futures.md b/_posts/2020-04-04-controllable-futures.md
new file mode 100644
index 000000000000..87bb008d46a0
--- /dev/null
+++ b/_posts/2020-04-04-controllable-futures.md
@@ -0,0 +1,105 @@
+---
+title: "Controllable Futures in Scala"
+date: 2020-04-04
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, async]
+excerpt: "In this article we learn to address the problem of \"deterministic\" Futures in Scala, using Promises."
+---
+In this article I'm going to address the problem of "deterministic" Futures. You probably know by now that Futures are inherently non-deterministic, in the sense that if you create a Future
+
+```scala
+val myFuture = Future {
+ // you have no future, you are DOOMED!
+ 42
+ // JK.
+}
+```
+
+you know the value inside will be evaluated on "some" thread, at "some" point in time, without your control.
+
+## The scenario
+
+Here I will speak to the following scenario which comes up in practice. Imagine you're designing an function of the following sort:
+
+```scala
+def gimmeMyPreciousValue(yourArg: Int): Future[String]
+```
+
+with the assumption that you're issuing a request to some multi-threaded service which is getting called all the time. Let's also assume that the service looks like this:
+
+```scala
+object MyService {
+ def produceThePreciousValue(theArg: Int): String = "The meaning of your life is " + (theArg / 42)
+
+ def submitTask[A](actualArg: A)(function: A => Unit): Boolean = {
+ // send the function to be evaluated on some thread, at the discretion of the scheduling logic
+ true
+ }
+}
+```
+
+So the service has two API methods:
+
+ 1) A "production" function which is completely deterministic.
+ 2) A submission function which has a pretty terrible API, because the function argument will be evaluated on one of the service's threads and you can't get the returned value back from another thread's call stack.
+
+Let's assume this important service is also impossible to change, for various reasons (API breaks etc). In other words, the "production" logic is completely fixed and deterministic. However, what's not deterministic is _when_ the service will actually end up calling the production function. In other words, you can't implement your function as
+
+```scala
+def gimmeMyPreciousValue(yourArg: Int): Future[String] = Future {
+ MyService.produceThePreciousValue(yourArg)
+}
+```
+
+because spawning up the thread responsible for evaluating the production function is not up to you.
+
+## The solution
+
+Introducing Promises - a "controller" and "wrapper" over a Future. Here's how it works. You create a Promise, get its Future and use it (consume it) with the assumption it will be filled in later:
+
+```scala
+// create an empty promise
+val myPromise = Promise[String]()
+// extract its future
+val myFuture = myPromise.future
+// do your thing with the future, assuming it will be filled with a value at some point
+val furtherProcessing = myFuture.map(_.toUpperCase())
+```
+
+Then pass that promise to someone else, perhaps an asynchronous service:
+
+```scala
+def asyncCall(promise: Promise[String]): Unit = {
+ promise.success("Your value here, your majesty")
+}
+```
+
+And at the moment the promise contains a value, its future will automatically be fulfilled with that value, which will unlock the consumer.
+
+## How to use it
+
+For our service scenario, here's how we would implement our function:
+
+```scala
+def gimmeMyPreciousValue(yourArg: Int): Future[String] = {
+ // create promise now
+ val thePromise = Promise[String]()
+
+ // submit a task to be evaluated later, at the discretion of the service
+ // note: if the service is not on the same JVM, you can pass a tuple with the arg and the promise so the service has access to both
+ MyService.submit(yourArg) { x: Int =>
+ val preciousValue = MyService.producePreciousValue(x)
+ thePromise.success(preciousValue)
+ }
+
+ // return the future now, so it can be reused by whoever's consuming it
+ thePromise.future
+}
+```
+
+So we create a promise and then we return its future at the end, for whoever wants to consume it. In the middle, we submit a function which will be evaluated at some point, out of our control. At that moment, the service produces the value and fulfils the Promise, which will automatically fulfil the Future for the consumer.
+
+This is how we can leverage the power of Promises to create "controllable" Futures, which we can fulfil at a moment of our choosing. The Promise class also has other methods, such as failure, trySuccess/tryFailure and more.
+
+I hope this was useful!
diff --git a/_posts/2020-04-06-higher-order-functions-for-oop.md b/_posts/2020-04-06-higher-order-functions-for-oop.md
new file mode 100644
index 000000000000..8022e27cd002
--- /dev/null
+++ b/_posts/2020-04-06-higher-order-functions-for-oop.md
@@ -0,0 +1,124 @@
+---
+title: "Higher-Order Functions for OO Programmers"
+date: 2020-04-06
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, functional programming]
+excerpt: "For OO programmers looking to dive into functional programming in Scala: a gentle introduction to functions working with other functions."
+---
+This article is for the programmer who is familiar with Scala concepts and structure, but has the object-oriented programming principles deeply ingrained. This article will not attempt to change them, but rather show you how you can map these principles to the very abstract functional programming concept of HOF, or higher-order functions.
+
+## Prelude
+
+You're probably aware that the apply method is treated in a special way:
+
+```scala
+class Applicable {
+ def apply(x: Int) = x + 1
+}
+
+val applicable = new Applicable
+
+applicable.apply(2) // 3
+applicable(2) // still 3
+```
+
+The apply method allows instances of classes to be "invoked" like functions. As such, objects with apply methods can behave like functions: they take arguments and return results. The Scala standard library actually has built-in types for function objects, which are nothing else but plain instances with apply methods:
+
+```scala
+val incrementer = new Function1[Int, Int] {
+ override def apply(x: Int) = x + 1
+}
+
+incrementer(4) // 5
+```
+
+Scala, being the nice functional language it is, allows for concise syntax sugars:
+
+```scala
+val incrementerAlt = (x: Int) => x + 1
+incrementerAlt(4) // 5 of course
+```
+
+The shorthand version is unwrapped by the compiler into the exact same Function1[Int, Int] construction which we saw earlier. The type of this function is Int => Int, which is also another sweet name for Function1[Int, Int].
+
+## The HOF baffle
+
+Naturally, because these "functions" are nothing but objects with apply methods, they can be passed around as arguments or returned as results. The functions which take other functions as arguments and/or return other functions as results are called HOFs, or higher-order functions. This is usually easy to make sense of.
+
+Here is something I often ask people to do in my trainings after I explain the above. Define a function which takes a function f and a number n, and returns another function whose implementation is f applied n times. In other words, write an implementation for
+
+```scala
+def nTimes(f: Int => Int, n: Int): Int => Int = ???
+
+//
+// If we call g = nTimes(f, 30), then
+// g(x) = f(f(f(...f(x)))) 30 times
+//
+```
+
+This is where I expect certain existing mental structures to either adapt or break, both of which are intentional effects I'm after. If you want to avoid spoilers, pause here and try this exercise yourself.
+
+Here's a possible implementation of this exercise:
+
+```scala
+def nTimes(f: Int => Int, n: Int): Int => Int =
+ if (n <= 0) (x: Int) => x
+ else (x: Int) => nTimes(f, n-1)(f(x))
+```
+
+## Um. Yeah, makes sense. But wait, wha...?
+
+The above code is concise and often hard to read, especially if you've not done a lot of this before, so it's natural if it takes a few minutes to unpack.
+
+_FAQ 1: How do you read this? _
+
+Let's take a look at the code. If n is zero or less, we return a function that given an argument returns the argument (the identity function). Otherwise, we return a function, that given an argument, _applies the n-1-times function to f(x)._. Look at this breakdown:
+
+```scala
+nTimes(f, 4) = x => nTimes(f, 3)(f(x))
+nTimes(f, 3) = x => nTimes(f, 2)(f(x))
+nTimes(f, 2) = x => nTimes(f, 1)(f(x))
+nTimes(f, 1) = x => nTimes(f, 0)(f(x))
+nTimes(f, 0) = x => x
+```
+
+So then
+
+```scala
+nTimes(f, 1) = x => nTimes(f, 0)(f(x)) = f(x)
+nTimes(f, 2) = x => nTimes(f, 1)(f(x)) = f(f(x))
+nTimes(f, 3) = x => nTimes(f, 2)(f(x)) = f(f(f(x)))
+nTimes(f, 4) = x => nTimes(f, 3)(f(x)) = f(f(f(f(x))))
+*/
+```
+
+_FAQ 2: When are these functions created? _
+
+If we read the code, we can see that all these intermediate functions are not created until we actually call the result function. For example, if we said
+
+```scala
+val f4 = nTimes(f, 4)
+```
+
+Then this will not create all the intermediate functions up to 0. It's as if I said
+
+```scala
+val f4 = (x: Int) => nTimes(f, 3)(f(x))
+```
+
+which is easier to see: this is not creating the rest of the functions up to n = 0. If we invoke f4, then that will be a different story, as all the intermediate functions will be created.
+
+_FAQ 3: I understand the mathematical definition. But to an OO programmer like me, how are these functions created in memory?_
+
+It's worth coming back to the origins of functions. Let me rewrite the code with the original types:
+
+```scala
+def nTimes(f: Function1[Int, Int], n: Int): Function1[Int, Int] =
+ if (n <= 0)
+ new Function1[Int, Int] { override def apply(x: Int) = x }
+ else
+ new Function1[Int, Int] { override def apply(x: Int) = nTimes(f, n-1).apply(f(x)) }
+```
+
+Somewhat counterintuitively, it's easier for OO (especially Java) programmers to read this, whereas the more experienced you are with Scala, the more bloated this code will seem. If you come from a very heavy OO background, this code will shed some light onto how the functions are getting created, because we're now talking about plain JVM objects. If you track this down in the same style we saw above, you will see how the function objects are being spawned in memory. Being recursive, these objects are short lived, so even though they might be using more memory than necessary, they will be quickly freed by the JVM's garbage collection.
diff --git a/_posts/2020-04-07-variables.md b/_posts/2020-04-07-variables.md
new file mode 100644
index 000000000000..6b8db74a8d1f
--- /dev/null
+++ b/_posts/2020-04-07-variables.md
@@ -0,0 +1,78 @@
+---
+title: "How Things Don't Make Sense - Scala Variables"
+date: 2020-04-07
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, teaching, rant]
+excerpt: "Daniel goes into a small rant about learning (and teaching) Scala using variables."
+---
+For a long time in my Scala classes and trainings I used to start the training, obviously, with values and variables.
+
+## The gripe
+
+This is what everyone starts with when learning Scala:
+
+```scala
+val aValue = 2
+aValue = 3 // NOPE. can't reassign
+
+var aVariable = 3
+aVariable = 47 // OK
+```
+
+Piece o' cake. This is what I used to start with as well. However, the very next thing I say is, "don't use variables, they are discouraged". People look confused, so I immediately follow with a "trust me" just to ease the tension in the room.
+
+The confusion is natural. People usually come from an imperative programming background, like Java or Python, where you can't do squat without variables. So from an instructor's point of view, introducing Scala by relying on something people already know seems natural and intuitive.
+
+I've recently started to believe that is the wrong approach.
+
+## Because FP
+
+Once someone gets started with Scala by learning mutable variables, they instantly validate every other concept they might have learned or used in other languages or thinking styles.
+
+```scala
+// do something 10 times
+var i = 0
+while (i < 10) {
+ println("Hey ma, I'm looping!")
+ i = i + 1
+}
+```
+
+In other words, people start thinking - or I should say continue thinking - procedurally. This is blocking the normal flow of learning functional programming, because we want think in terms of expressions, not instructions.
+
+```scala
+(0 until 10).foreach(_ => println("Hey ma, I'm doing it right!"))
+```
+
+The looping version above would not pass code review in any strong Scala team.
+
+Mutation is also a big problem:
+
+```scala
+var myList = List(1,2,3)
+// use the list
+myList = myList :+ 4
+// pass the list to some other API
+invokeExternalService(myList)
+```
+
+Even if Scala learners understand immutable data structures, reusing variables makes code very hard to read and reason about, especially in multithreaded or distributed applications. In this case, we can very quickly lose track of who modified which variable, or which piece of code reassigned our shared variables. If we start learning Scala with variables, we continue thinking this way, and make concurrent and distributed code worse in Scala than in Java. Instead, keep it clean.
+
+```scala
+val startingList = List(1,2,3)
+// use it
+// ...
+// pass something else to the other API
+invokeExternalService(startingList :+ 4)
+```
+
+That's not to say that variables are bad all the time. I'll probably talk about when variables are useful in another article. But if we want to discourage people from using variables, why are variables the first thing they learn?
+
+## The contrarian
+
+So I've stopped teaching people about variables, and my recent trainings at Adobe and the Scala at Light Speed embody that belief. I start with values and just carry on with expressions, then functions and then teach people how to build moroe complex things. Some trainees and students sometimes ask me, "Daniel, isn't there some variable or something?" at which I tell a lie and say "Nope. You'll have to use what you have.". And they always do, they find a way, and after the training or the class they tell me they understand Scala as a different way of thinking, which is my main goal. Only then do I tell them that there is such a thing as a variable in Scala, but by that time, nobody even cares.
+
+Rule of thumb:
+ - If you're learning Scala, pretend you've never even heard of variables. There are no such things.
+ - If you're teaching Scala, if you don't want your students to use variables and loops, don't teach them!
diff --git a/_posts/2020-04-08-reading-query-plans.md b/_posts/2020-04-08-reading-query-plans.md
new file mode 100644
index 000000000000..7a4ba323ea3b
--- /dev/null
+++ b/_posts/2020-04-08-reading-query-plans.md
@@ -0,0 +1,148 @@
+---
+title: "Reading Spark Query Plans"
+date: 2020-04-08
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [spark, how to]
+excerpt: "In this article you'll learn one of the most important Spark skill: reading how your job will run. This is foundational to any further Spark optimization."
+---
+This article is for the Spark programmer who has at least some fundamentals, e.g. how to create a DataFrame and how to do basic operations like selects and joins, but has not dived into how Spark works yet. Perhaps you're interested in boosting the performance out of your Spark jobs.
+
+## Prerequisites
+
+The code I'll be writing is inside a Spark shell with version 3.0.0, which you can find here for download. The default choices in the dropdowns will give you a pre-compiled Spark distribution. Just download, unzip, navigate to the bin folder, then run the spark-shell executable.
+
+That is, if you've never installed Spark before.
+
+## How Spark doesn't run
+
+You're probably aware that Spark has this lazy execution model, i.e. that you write transformations but they're not actually run until you call an action, like a show, or collect, or take, etc.
+
+When you write transformations, Spark will automatically build up a dependency tree of your DataFrames, which will actually end up executing when you call an action. Let me give an example:
+
+```scala
+val simpleNumbers = spark.range(1, 1000000)
+val times5 = simpleNumbers.selectExpr("id * 5 as id")
+```
+
+That times5 DataFrame will not actually get evaluated. If you hit those two lines in the Spark shell, you'll notice that they return instantly. However, when you do
+
+```scala
+times5.show()
+```
+
+then this will take a little more than an instant - that's because Spark is only now performing the actual computations. If you want to see what operations Spark did, you can try explaining the DF:
+
+```scala
+times5.explain()
+```
+
+and this will give you the prettiest thing:
+
+```Perl
+== Physical Plan ==
+*(1) Project [(id#0L * 5) AS id#2L]
++- *(1) Range (1, 1000000, step=1, splits=6)
+```
+
+This is a query plan. When you call the explain method, the final plan - the physical one that will actually be run on executors - will be shown here. You can inspect this plan for your massive computations before kicking off any job. This is particularly important because the ability to read and interpret query plans allows you to predict performance bottlenecks.
+
+## Reading plans
+
+So let's go over some examples of query plans and how to read them. Let's go back to the one we've just shown:
+
+```Perl
+== Physical Plan ==
+*(1) Project [(id#0L * 5) AS id#2L]
++- *(1) Range (1, 1000000, step=1, splits=6)
+```
+
+We read this plan backwards, bottom to top:
+
+1) First Spark builds a Range object from 1 to 1000000. The splits parameter defines the number of partitions.
+2) Then Spark does a "project", which is the mathematical term for a database/DF select. So we're selecting the column id with the identifier 0L (long type), and multiplying the value there with 5. The result will be a new column with the name id and the identifier 2.
+
+Or we can read this plan top to bottom, like this:
+
+2) The end of the computation is a "project" (with the meaning above), which depends on
+1) A Range from 1 to 1 million in 6 partitions.
+
+Not too hard. Let's do another:
+
+```scala
+val moreNumbers = spark.range(1, 10000000, 2)
+val split7 = moreNumbers.repartition(7)
+split7.explain()
+```
+
+```Perl
+== Physical Plan ==
+Exchange RoundRobinPartitioning(7)
++- *(1) Range (1, 10000000, step=2, splits=6)
+```
+
+Same operation first, but the next step is an Exchange, which is another name for a shuffle. You're probably aware - a shuffle is an operation in which data is exchanged (hence the name) between all the executors in the cluster. The more massive your data and your cluster is, the more expensive this shuffle will be, because sending data over takes time. For performance reasons, it's best to keep shuffles to a minimum.
+
+So a performance tip: whenever you see Exchange in a query plan, that's a perf bottleneck.
+
+Let's do one more, this time make it complex:
+
+```scala
+val ds1 = spark.range(1, 10000000)
+val ds2 = spark.range(1, 10000000, 2)
+val ds3 = ds1.repartition(7)
+val ds4 = ds2.repartition(9)
+val ds5 = ds3.selectExpr("id * 5 as id")
+val joined = ds5.join(ds4, "id")
+val sum = joined.selectExpr("sum(id)")
+sum.explain
+```
+
+```Perl
+== Physical Plan ==
+*(7) HashAggregate(keys=[], functions=[sum(id#36L)])
++- Exchange SinglePartition
+ +- *(6) HashAggregate(keys=[], functions=[partial_sum(id#36L)])
+ +- *(6) Project [id#36L]
+ +- *(6) SortMergeJoin [id#36L], [id#32L], Inner
+ :- *(3) Sort [id#36L ASC NULLS FIRST], false, 0
+ : +- Exchange hashpartitioning(id#36L, 200)
+ : +- *(2) Project [(id#30L * 5) AS id#36L]
+ : +- Exchange RoundRobinPartitioning(7)
+ : +- *(1) Range (1, 10000000, step=1, splits=6)
+ +- *(5) Sort [id#32L ASC NULLS FIRST], false, 0
+ +- Exchange hashpartitioning(id#32L, 200)
+ +- Exchange RoundRobinPartitioning(9)
+ +- *(4) Range (1, 10000000, step=2, splits=6)
+```
+
+Now that's a nasty one. Let's take a look.
+
+Notice this plan is now branched. It has two sections:
+
+section 1, bottom:
+ A Range 1 to 1 million, in steps of 2, in 6 partitions,
+ which is repartitioned (shuffled) into 9 partitions,
+ then repartitioned again by a hash partitioner with the key being the column id (identifier 32) into 200 partitions,
+ then sorted by the column id (identifier 32) ascending.
+
+section 2, less bottom:
+ A range 1 to 1 million, in steps of 1, in 6 partitions,
+ repartitioned (shuffled) to 7 partitioned,
+ modified as id * 5, with the name id (identifier 36)
+ then repartitioned again by a hash partitioner with the key being the column id (identifier 36) into 200 partitions
+ then sorted by that
+
+Then those two sections are used for the SortMergeJoin. The join operation requires that the DFs in question be partitioned with the same partitioning scheme and sorted. This is why Spark does all these operations.
+
+Then we have a project (select), in which we're selecting just one of the two columns, because otherwise we'd have duplicate columns.
+
+Then we have an operation called HashAggregate, in which partial sums are being computed on every partition.
+
+Then, all the partial sums are brought to a single partition by the last exchange.
+
+Finally we have a single HashAggregate operation in which all the partial sums are now combined for a single result.
+
+## Just the beginning
+
+This seems tedious, but in practice, the skill of reading and interpreting query plans is invaluable for performance analysis. You might notice that in the last example, we're doing quite a few shuffles. With time, you will learn to quickly identify which transformations in your code are going to cause a lot of shuffling and thus performance issues.
diff --git a/_posts/2020-04-14-the-brilliance-of-materialized-values.md b/_posts/2020-04-14-the-brilliance-of-materialized-values.md
new file mode 100644
index 000000000000..2e3b928d4ccc
--- /dev/null
+++ b/_posts/2020-04-14-the-brilliance-of-materialized-values.md
@@ -0,0 +1,124 @@
+---
+title: "Materialized Values in Akka Streams"
+date: 2020-04-14
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka, akka streams]
+excerpt: "We demystify one of the hardest concepts in Akka Streams: materialized values."
+---
+This article is for the Scala programmer who has the very very basic familiarity with Akka Streams, in the sense that you can use some already made components, glue them together and start the stream, but otherwise no knowledge of behind-the-scenes functionality is required. If you've never used Akka Streams, I'll give a brief overview.
+
+## The setup
+
+If you are curious enough to try this code for yourself, you'll need to start an SBT project. I recomment IntelliJ because it's easiest. Add the following to your build.sbt file:
+
+```scala
+val akkaVersion = "2.6.4"
+
+libraryDependencies ++= Seq(
+ // akka streams
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion
+)
+```
+
+## A 30-Second Background
+
+Akka Streams is an implementation of the Reactive Streams specification for the JVM. It allows high-throughput and fault-tolerant streams of data by simply plugging in streaming components. They are sources (or publishers), flows (or transformers) and sinks (or subscribers). The names are self-explanatory: one produces elements, one transforms them along the way, and the last one consumes the data.
+
+You can create streams of data in the following way:
+
+```scala
+import akka.actor.ActorSystem
+import akka.stream.scaladsl.{Sink, Source}
+
+// Akka Streams runs on top of actors for high-throughput
+implicit val system = ActorSystem()
+
+// streaming components
+val source = Source(1 to 1000) // can wrap a normal collection and "emit" the elements one at a time
+val flow = Flow[Int].map(x => x * 2) // transforms every incoming x into 2 * x
+val sink = Sink.foreach[Int](println) // applies this function to every element that goes into it
+val graph = source.via(flow).to(sink) // plug them together
+
+// magic
+graph.run() // prints 2, 4, 6, 8, ... on different lines
+```
+
+Obviously, there is a whole variety of sources, sinks and flows, but you can plug them all together in a myriad of ways and construct arbitrary pipes of data through which you can send elements.
+
+## The Confusion
+
+Now, if you take a look at the types of the streaming elements in question, you will see the following (with the help of the compiler):
+
+```scala
+val source: Source[Int, NotUsed] = ...
+val flow = Flow[Int, Int, NotUsed] = ...
+val sink = Sink[Int, Future[Done]] = ...
+```
+
+The types there are confusing, especially for a newcomer. It's quite understandable if you anticipated a source to be of type Source[Int] and a Flow to be of type Flow[Int, Int] much like a function, but what's the third type doing? What's the NotUsed and the Future[Done]?
+
+At this point many Akka Streams programmers either ignore the third type and focus on what they understand, or try to read the docs and become entangled in a web of abstractions which have no correspondent in other code. Some of them do succeed, though, sometimes through reading the docs many many many (x10) times. It doesn't have to be hard, and that third type is actually pretty damn great. Let me explain.
+
+## The Jedi Values
+
+It's one thing to run a stream: plug the stream components together and you obtain what is called a graph (pretty understandable), then you call the run method and the graph now has its own life.
+
+But what kind of value do you get when you run a graph? Running a graph is an expression, much like anything else in Scala, so it must have a type and a value. What's the value of running a graph?
+
+I'm going to call that a Jedi value, for reasons that (I hope) will become apparent shortly. A Jedi value is what you obtain after running a graph. When we run
+
+```scala
+source.via(flow).to(sink).run()
+```
+
+the Jedi value of this graph is of type NotUsed, which is a bit like Unit and not useful for processing. Now here's the thing with Jedi values: ALL streaming components in Akka Streams have a Jedi value when plugged into a living breathing graph, so in the above code we have 3 components and thus 3 Jedi values somewhere. However, the graph itself can only return ONE. So then the question becomes, which Jedi value does the graph return?
+
+When you run something like source.via(flow).to(sink), the left-most Jedi value will be picked. In our case, the Jedi value of the source. You can change that.
+
+```scala
+source.via(flow).toMat(sink)((leftJediValue, rightJediValue) => rightJediValue).run()
+```
+
+By calling the toMat method, you can pass a function that takes the left Jedi value before the last dot, and the sink's Jedi value, and return something from them. In this case I'm returning just the right Jedi value. By calling toMat, I've changed the Jedi value of the graph, which will now take the Jedi value of the sink. You can use toMat in between any streaming operator to choose which Jedi value you want.
+
+In this case above, the Jedi value of the graph is now the Jedi value of the sink, which is Future[Done]. So now you can use it:
+
+```scala
+import system.dispatcher
+val future = source.via(flow).toMat(sink)((leftJediValue, rightJediValue) => rightJediValue).run()
+future.onComplete(_ => println("Stream is done!"))
+```
+
+Notice that the Jedi value has a meaning , which is that the future can be monitored for completion and you can tell when the stream has finished running. Also notice that the Jedi value has no connection whatsoever with the elements being shoved into the stream. That function that picks the right argument from the two arguments supplied to it can also be written as:
+
+```scala
+source.via(flow).toMat(sink)(Keep.right).run() // Keep.right is exactly (a, b) => b
+```
+
+or even shorter
+
+```scala
+source.via(flow).runWith(sink) // Keep.right is assumed
+```
+
+There are various components in Akka Streams that return various Jedi values. For example, if you want to extract the sum of all elements in a stream:
+
+```scala
+val summingSink = Sink.fold[Int, Int](0)((currentSum, newNumber) => currentSum + newNumber)
+// remember from the previous snippet: runWith takes the rightmost Jedi value
+val sumFuture = source.runWith(summingSink)
+sumFuture.foreach(println)
+```
+
+Not only will you know when the future is done, but you will also know what the sum of the elements was. Notice that in this case we can use the elements in the stream to process them in a single value that the stream will return at the end (the Jedi value).
+
+## Why are Jedi values great?
+
+Jedi values are awesome, because without them, once you start the graph, your pipes are opaque, sealed and irreversible. There's no way of controlling streams - I'll talk another time about how to shove data into a stream manually, or as a reaction to something. There's no getting any information out of the stream. There's no processing the values aside from flows. The world would be a very dark place.
+
+Various streaming components in Akka streams can offer various Jedi values. Sinks usually offer Futures, which often allow you to combine the values inside the streams into one, and tell if/when the stream has finished. Some flows offer control mechanisms - like KillSwitches which allow you to stop the stream at any point. Some sources offer entry points through which you can send the data inside the stream.
+
+I call them Jedi values because they are powerful. The Akka Streams library calls them materialized values . That's because, when you plug components together, you have an inert graph, but when you call the run method, the graph comes alive, or is materialized . The Jedi value returned by materializing a graph is called a materialized value. The value may or may not be connected to the actual elements that flow through the stream, and the value can be of any type - which again, may or may not be different from the types of elements flowing through the graph.
+
+I hope this article saved you the many hours it took me to internalize this abstract and often confusing concept.
diff --git a/_posts/2020-04-15-scala-looping.md b/_posts/2020-04-15-scala-looping.md
new file mode 100644
index 000000000000..145a06e84c1d
--- /dev/null
+++ b/_posts/2020-04-15-scala-looping.md
@@ -0,0 +1,126 @@
+---
+title: "How Things Don't Make Sense - Scala Loops"
+date: 2020-04-15
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, teaching, rant]
+excerpt: "Daniel goes into another rant about learning (and teaching) Scala using loops. Are they actually useful?"
+---
+I wrote an article not too long ago on why learning variables in Scala doesn't make sense , and likewise teaching variables as one of the first concepts also doesn't make sense. In this article I'm going to expand that idea to the lovely loops.
+
+This article is for 1) programmers who are just getting started with Scala and 2) for Scala teachers.
+
+## What many start with
+
+When learning Scala, it's very attractive, intuitive and tempting to start with what people already know. After all, Scala is mostly targeted at established programmers in other languages (especially Java), as very few instructors have attempted teaching Scala as a first language. I've yet to make this attempt myself - but I digress. So when learning Scala, people start with the familiar:
+
+```scala
+// values are constants
+val x = 3
+
+// variables are changeable, much like any other language
+var y = 4
+y = 5 // reassignment ok
+
+// looping
+while (y < 42) {
+ println("Hey ma, I'm looping!")
+ y += 1
+}
+```
+
+After that, the instructor usually says: "Cool, now that you've learned about while, please don't use them. It's bad practice.". People look confused, so the instructor continues: "Just trust me.". For a long time, I've been guilty of this myself. I even continued further: "I've just shown you loops so that you can relate to them, but please don't use them.".
+
+What I was expecting from the audience: "OK, we've wiped my memory clean and fresh. We've never heard of a loop in our life. Show us the ways of the Force."
+What I actually got: What should we use instead? If you don't want us to use them, why are you showing us this?
+
+Exactly.
+
+## Make FP, not war
+
+Learning loops to get familiar with Scala is bad. It's as if you wanted to learn French but still pronounce the 'h'. You need to let it go.
+
+If loops are one of the first things you learn in Scala, that will validate all the other concepts you might have encountered in other languages. You will continue to think imperatively, that is, "do this, do that, increment this, and as long as y < 42, just increment y". In Scala, we think in terms of expressions, not instructions:
+
+```scala
+(5 until 42).foreach(_ => println("Hey ma, I'm doing it right!"))
+```
+
+If you want to transform a list, you don't use a loop, you use a map:
+
+```scala
+List(1,2,3).map(x => x + 1)
+```
+
+If every element you go through generates its own collection, you use a flatMap:
+
+```scala
+List(1,2,3).flatMap(n => Seq.fill(n)("I like FP!"))
+```
+
+What if you don't like all the elements in your collection? Use a filter:
+
+```scala
+(1 to 10000).filter(n => n % 42 == 0)
+```
+
+Want a single value out of the entire list? Use fold, count, maxBy, find and a variety of other transformations. You see, every "loop" has an equivalent transformation. Newbies ask "how can I loop through this?". Terrible, albeit understandable question. Leads to ugly, unproductive code which will not pass any code review in a mature Scala team. Instead, ask "how can I transform this into what I want?". That's a better question. Leads to clean, elegant code that will stand the test of time and will help fellow developers push more robust code faster.
+
+## The foreach fallacy
+
+"But Daniel, I can still loop with foreach!"
+
+That's one of the unfortunate confusions many starters face. Foreach appears in many programming languages in various forms, so assuming a foreach in Scala is a built-in language features is understandable. It's easy to consider
+
+```scala
+List(1,2,3).foreach { x =>
+ println(x)
+}
+```
+
+as being similar to
+
+```Java
+List list = ...
+for (int x: list) {
+ System.out.println(x)
+}
+```
+
+But it's not. Foreach is not built into the Scala language. Foreach is one of the higher-order functions that are part of every standard collection. It's particularly confusing given Scala's alternative lambda syntax with curly braces, but the "x =>" that follows the curly braces is an anonymous function, nothing more.
+
+## The for "loop"
+
+And don't get me started with for comprehensions.
+
+```scala
+for {
+ x <- List(1, 2, 3)
+ y <- List('a', 'b', 'c')
+} yield (x, y)
+```
+
+Another (understandable) confusion about the Scala language. Now, for-structures like the one above ARE built into the language. It's just that they're not what they seem:
+
+```scala
+List(1, 2, 3).flatMap(x => List('a', 'b', 'c').map(y => (x, y)))
+```
+
+That's what the for "loop" above compiles to. That's why for-structures in Scala are called comprehensions. They're expressions, because they can be assigned to a value:
+
+```scala
+val allPairs = for {
+ x <- List(1, 2, 3)
+ y <- List('a', 'b', 'c')
+} yield (x, y)
+```
+
+## To the Scala learner
+
+If you heard about loops in Scala, try to forget them. Try to do without them. Instead of asking "how can I loop through this", ask "how can I transform this". Take this one principle to heart and you'll be ahead of many Scala programmers already.
+
+## To the Scala teacher
+
+Why are we teaching loops in Scala so early? It makes no sense. We want people to think in FP, with values, expressions, recursion and higher-order functions. So why are loops the first thing they learn?
+
+Roughly a year ago, I personally stopped even mentioning loops until people are already familiar with the different mindset of functional programming. After enough practice with FP, I tell them "oh, by the way, there's also the while loop in Scala" at which they reply "that's fine, we don't need it".
diff --git a/_posts/2020-04-21-nothing.md b/_posts/2020-04-21-nothing.md
new file mode 100644
index 000000000000..3585a144d259
--- /dev/null
+++ b/_posts/2020-04-21-nothing.md
@@ -0,0 +1,100 @@
+---
+title: "Much Ado About Nothing in Scala"
+date: 2020-04-21
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "We go on a short trip to Nothing (and nothing-ness) in Scala. Nothing is actually pretty useful."
+---
+I've seen quite some confusion about the Nothing type in Scala and I wanted to shed some light on it. If you're a Scala programmer, you might have seen Nothing once or twice, and you have at least the broad picture of Scala's type hierarchy.
+
+
+## Quick intro
+
+the article where I discuss Spark query plans , after many people requested I follow the Spark path and shed some light on other Spark functionality.
+
+## Prerequisites
+
+The code I'll be writing is inside a Spark shell with version 3.0.0, which you can find here for download. The default choices in the dropdown selectors will give you a pre-compiled Spark distribution. Just download, unzip, navigate to the bin folder, then run the spark-shell executable.
+
+That is, if you've never installed Spark before.
+
+## The intro
+
+You're surely aware that Spark has this lazy execution model, i.e. that you write transformations but they're not actually run until you call an action, like a show, or collect, or take, etc.
+
+When you write transformations, Spark will automatically build up a dependency graph of your DataFrames, which will actually end up executing when you call an action.
+
+```scala
+val simpleNumbers = spark.range(1, 1000000)
+val times5 = simpleNumbers.selectExpr("id * 5 as id")
+```
+
+That times5 DataFrame will not actually get evaluated until you call an action, like
+
+```scala
+times5.show()
+```
+
+Only at this point will Spark be performing the actual computations. An action will trigger a Spark job, which will be visible in the Spark UI. If you run this locally, either in your IDE or on your Spark Shell, usually the Spark UI will be at localhost:4040 . When you go to the Spark UI, you'll see a table with all the jobs that the application has completed and is currently running. If you click on the one you just ran, you'll see something like this:
+
+while loops . Although this could be way uglier, we can do better.
+
+## Version 2: Java-style, friendlier, with cheats
+
+Use a widely used library called Apache Commons and add this to your build.sbt file:
+
+```scala
+libraryDependencies += "commons-io" % "commons-io" % "2.6"
+```
+
+After that, reading a file should be a piece of cake. Import first:
+
+```scala
+import org.apache.commons.io.FileUtils
+```
+
+and then:
+
+```scala
+val fileContents = FileUtils.readLines(file) // a list of lines that you can now process freely
+```
+
+This returns a Java list of lines that you can now process as you see fit. It's much better, because you can now operate with values and expressions rather than variables and instructions . However, still not enough as this is a Java collection, which doesn't have the necessary niceties that play with Scala so well.
+
+## Version 3: the Scala way
+
+No importing some external library. Use a reader called Source:
+
+```scala
+import scala.io.Source
+```
+
+which comes pre-bundled with Scala. Then, one line:
+
+```scala
+val scalaFileContents = Source.fromFile(file).getLines()
+```
+
+This now returns an Iterator of the lines in the file. This time, this is a Scala collection, which you can apply map, flatMap, filter, toList, mkString and all the niceties we're used to. The best part is that this iterator is not fully loaded in memory, so unlike version 2, you can read the file slowly rather than load everything in memory and then disposing of the contents.
+
+## Version 4: like a boss
+
+Why can't Scala read a damn file like Python? Just "open" and then "read". Rolling sleeves, and we can:
+
+```scala
+def open(path: String) = new File(path)
+
+implicit class RichFile(file: File) {
+ def read() = Source.fromFile(file).getLines()
+}
+```
+
+So we defined a method to obtain a File object by opening the file at the given path. Then we're bringing the big guns and we're enhancing the File type with an implicit class. So when we call read() on a File object, normally the code wouldn't compile, but the compiler will be nice and will wrap the File object into a new instance of RichFile. So we can now say
+
+```scala
+val readLikeABoss = open(filePath).read()
+```
+
+## Epilogue
+
+In the JVM world - Java in particular, but including Scala - we're still far away from having simple, elegant, intuitive and friendly standard APIs, especially for beginners who just need to read a simple file.
diff --git a/_posts/2020-05-08-sync,-async-and-(non)-blocking.md b/_posts/2020-05-08-sync,-async-and-(non)-blocking.md
new file mode 100644
index 000000000000..cefc80ae94e6
--- /dev/null
+++ b/_posts/2020-05-08-sync,-async-and-(non)-blocking.md
@@ -0,0 +1,114 @@
+---
+title: "Sync, Async and (non) blocking in Scala and Akka"
+date: 2020-05-08
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, akka]
+excerpt: "We discuss the tradeoffs between 3 different styles of writing parallel code, in terms of thread use (and other effects) in Scala and Akka."
+---
+This article is for programmers of all levels, willing to deal with asynchronous and/or non-blocking computation. I'm going to write Scala (naturally), but the problem I'll address is general and across almost every language and tech: the difference between synchronous, asynchronous, blocking and non-blocking.
+
+## Synchronous, blocking
+
+Without invoking any other threads, every single thing we write is serial. However, not all things are active in the sense that they perform work. Our job is to get the most out of our CPU, but some function calls invoke some sort of resource (like a database), or wait for it to start, or wait for a response. That is called a blocking call. Blocking, because when you call the function, you can't do anything until you get a result.
+
+```scala
+def blockingFunction(arg: Int): Int = {
+ Thread.sleep(10000)
+ arg + 42
+}
+```
+
+If we call the function, the subsequent expression will need to wait at least 10 seconds:
+
+```scala
+blockingFunction(3)
+val theMeaningOfLife = 42 // I don't want to touch on the philosophical, but this happens in real life.
+```
+
+The main downside of blocking calls is that the calling thread is neither doing any work, nor is it making any progress, nor is it yielding control to something else.
+
+## Asynchronous, blocking
+
+If a synchronous computation performs serially, then an asynchronous expression will be evaluated in parallel. So while your main program flow carries on, your async computation also runs at the same time. This happens, of course, due to both multi-processor systems - which allow us to perform multiple computations at literally the same time - or because of smart process scheduling - which moves so fast that it gives us the impression of multiple things happening simultaneously. In Scala, an asynchronous computation is called a Future, and it can be evaluated on another thread.
+
+```scala
+def asyncBlockingFunction(arg: Int): Future[Int] = Future {
+ Thread.sleep(10000)
+ arg + 42
+}
+```
+
+In this case, when we call this method
+
+```scala
+asyncBlockingFunction(3)
+val theMeaningOfLife = 42
+```
+
+The value after the call evaluates immediately, because the actual computation of the async method will run in parallel, on another thread. This is asynchronous. However, it's also blocking because, although you're not blocking the main flow of the program, you're still blocking some thread at (or close to) the moment you're calling the method. It's like passing the burning coal from your hand to someone else.
+
+The blocking aspect comes from the fact that these kinds of computations need to be constantly monitored for completion. It's like you spawn an annoying parrot, saying:
+
+"Are you done?"
+"Are you done?"
+"Are you done?"
+"Are you done?"
+"How about now?"
+
+so that when you are indeed done, the parrot will say "roger that" and will fly back to you (the calling thread) to deliver the result.
+
+## Asynchronous, non-blocking
+
+The true non-blocking power comes from actions that do not block either you (the calling thread) or someone else (some secondary thread). Best exemplified with an Akka actor. An actor, unlike what you may have read from the webs, is not something active. It's just a data structure. The power of Akka comes from the fact that you can create a huge amount of actors (millions per GB of heap), so that a small number of threads can operate on them in a smart way, via scheduling.
+
+```scala
+def createSimpleActor() = Behaviors.receiveMessage[String] { someMessage =>
+ println(s"Received something: $someMessage")
+ Behaviors.same
+}
+```
+
+Here I'm using the Akka Typed API. The short story is that the above describes what an actor will do: given a message of type String, it will print something out, and the actor will resume to its same behavior. The API is a bit obscure, and I might talk about it another time. If we create an actor:
+
+```scala
+val rootActor = ActorSystem(createSimpleActor(), "TestSystem") // guardian actor that will create an entire hierarchy
+rootActor ! "Message in a bottle"
+```
+
+Then calling the tell method (!) on the actor is completely asynchronous _and non-blocking_. Why non-blocking? Because this doesn't block the calling thread - the tell method returns immediately - and also because it doesn't spawn (or block) any other thread. Because Akka has an internal thread scheduler, it will be some point in the future when a thread will be scheduled to dequeue this message out of the actor's mailbox and process it for me.
+
+Asynchronous, non-blocking computation is what you want.
+
+However, even in this example we have a drawback: we aren't returning any meaningful value out of the interaction. To solve that, we could return a Future which the actor might complete manually. Check out the controllable Futures article for details into the reasoning.
+
+```scala
+ val promiseResolver = ActorSystem(
+ Behaviors.receiveMessage[(String, Promise[Int])] {
+ case (message, promise) =>
+ // do some computation
+ promise.success(message.length)
+ Behaviors.same
+ },
+ "promiseResolver"
+ )
+```
+
+This actor will complete a promise when it receives a message. On the other end - in the calling thread - we could process this promise when it's complete. Let's define some sensible API that would wrap this asynchronous, non-blocking interaction:
+
+```scala
+def doAsyncNonBlockingThing(arg: String): Future[Int] = {
+ val aPromise = Promise[Int]()
+ promiseResolver ! (arg, aPromise)
+ aPromise.future
+}
+```
+
+Here's how we could use it:
+
+```scala
+val asyncNonBlockingResult = doAsyncNonBlockingThing("Some message")
+asyncNonBlockingResult.onComplete(value => s"I've got a non-blocking async answer: $value")
+```
+
+In this way, neither the calling thread, nor some other thread is immediately used by the call, and we still return meaningful values from the interaction, which we can register a callback on when complete. For some reason, the ask pattern in Akka Typed is very convoluted, but on that, another time.
diff --git a/_posts/2020-05-11-Akka-Typed-incentivizes-you-to-write-good-code.md b/_posts/2020-05-11-Akka-Typed-incentivizes-you-to-write-good-code.md
new file mode 100644
index 000000000000..503940acabff
--- /dev/null
+++ b/_posts/2020-05-11-Akka-Typed-incentivizes-you-to-write-good-code.md
@@ -0,0 +1,113 @@
+---
+title: "How Akka Typed Incentivizes You to Write Good Code"
+date: 2020-05-11
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka]
+excerpt: "Akka Typed is one of the best things that happened with the Akka API. We explore how good practices are baked into the API directly."
+---
+Akka Typed is widely praised for bringing compile-time checks to actors and a whole new actor API. The problem is that even this new typed API has loopholes that almost never completely close the old Akka anti-patterns. However, the 2.6 API is a big step in the right direction, because although the API is not airtight, it's extremely powerful in the way that it shapes incentives to write good code. Let me give some examples for some of the most basic things.
+
+## Typed messages
+
+Let me start with the obvious: the messages an actor can receive will be reflected in the type of the actor, and vice-versa: the type of its ActorRef will be both a reason for the compiler to yell at you if you don't send a message of the right type, and also an indication to the user of the actor about what it's supposed to do. Let me give an example:
+
+```scala
+ trait ShoppingCartMessage
+ case class AddItem(item: String) extends ShoppingCartMessage
+ case class RemoveItem(item: String) extends ShoppingCartMessage
+ case object ValidateCart extends ShoppingCartMessage
+
+ val shoppingActor = ActorSystem(
+ Behaviors.receiveMessage[ShoppingCartMessage] { message =>
+ message match {
+ case AddItem(item) =>
+ println(s"adding $item")
+ case RemoveItem(item) =>
+ println(s"removing $item")
+ case ValidateCart =>
+ println("checking cart")
+ }
+
+ Behaviors.same
+ },
+ "simpleShoppingActor"
+ )
+```
+
+Please ignore that I'm catastrophically using Double for currency, and don't try this at home (or in production). The problem with the new typed behaviors is that messages are still pattern-matched, and it's very unlikely that in your big-ass application an actor will process a single message type EVER. However, the natural tendency is to think of an actor as receiving a message from a given hierarchy, which leads to a nice OO-type structure of messages. Of course, you can circumvent this and use Any, but why would you do that? Right from the moment of typing Any there you probably get an icy feel in the back of your neck that there's something wrong with your code.
+
+Additionally, if your message hierarchy is sealed, then the compiler will also help you treat every case.
+
+## Mutable state
+
+Mutable state was discouraged in Akka from the very beginning. If you remember the old "classic" API, we had this context-become pattern to change actor behavior and hold immutable "state" in method arguments returning receive handlers. Variables and mutable state have not disappeared:
+
+```scala
+ val shoppingActorMutable = ActorSystem(
+ Behaviors.setup { _ =>
+ var items: Set[String] = Set()
+
+ Behaviors.receiveMessage[ShoppingCartMessage] {
+ case AddItem(item) =>
+ println(s"adding $item")
+ items = items + item
+ Behaviors.same
+ case RemoveItem(item) =>
+ println(s"depositing $item")
+ items = items - item
+ Behaviors.same
+ case ValidateCart =>
+ println(s"checking cart: $items")
+ Behaviors.same
+ // can also try with pattern matching and returning Behavior.same once
+ }
+ },
+ "mutableShoppingActor"
+ )
+```
+
+However, in most Behavior factories - if not all, I'm not 100% up to speed yet - you are forced to return a new behavior after a message is being handled. In other words, the behavior changing is baked into the API now. With this in mind, it's much easier to create different behaviors and have the actors adapt in a more logical way:
+
+```scala
+ def shoppingBehavior(items: Set[String]): Behavior[ShoppingCartMessage] =
+ Behaviors.receiveMessage[ShoppingCartMessage] {
+ case AddItem(item) =>
+ println(s"adding $item")
+ shoppingBehavior(items + item)
+ case RemoveItem(item) =>
+ println(s"removing $item")
+ shoppingBehavior(items - item)
+ case ValidateCart =>
+ println(s"checking cart: $items")
+ Behaviors.same
+ }
+```
+
+So why would you need variables anymore when you have this logical code structure that avoids mutable state altogether? Much easier to write good code.
+
+## Actor hierarchy
+
+One of the massive benefits of Akka was the "let it crash" mentality embedded into the toolkit. This was achieved by making the actors maintain a supervision hierarchy, in which if an actor fails, then its parent - which acts like a supervisor - can deal with the failure and decide whether to restart the actor, stop it, resume it or simply escalate to its parent.
+
+A common anti-pattern of the old Akka API was spawning very flat hierarchies, which destroyed this massive benefit. The crux of the problem was the easily usable system.actorOf. Everyone anywhere could go "system.actorOf" left and right and all of a sudden you had massive groups of actors managed by the same user guardian actor with the default supervision strategy.
+
+In the new API, we don't have that. No more system.actorOf. You are now forced to think of the actor hierarchy and how the root guardian will manage them. That's simply by the fact that you can't spawn actors AT ALL - you can only spawn child actors:
+
+```scala
+ val rootActor = ActorSystem(
+ Behaviors.setup { ctx =>
+ // create children
+ ctx.spawn(shoppingBehavior(Set()), "danielsShoppingCart")
+ // no behavior in the root actor directly
+ Behaviors.empty
+ },
+ "onlineStore"
+ )
+```
+
+So even if your root actor doesn't handle messages itself, the fact that you can only spawn actors from a hierarchy is a huge win. It forces you to be a good citizen and embed actor hierarchy - and thus supervision - into your code.
+
+## A way forward
+
+This new API redefines what "normal" code should look like, and for the most part, it's shepherding Akka code towards the right direction. As I mentioned earlier, the API is not airtight and it can still be circumvented, but expect to see better Akka code in the future simply by the tools we now have at our disposal.
diff --git a/_posts/2020-05-12-how-to-create-your-own-string-interpolator.md b/_posts/2020-05-12-how-to-create-your-own-string-interpolator.md
new file mode 100644
index 000000000000..eb7a80ac2d14
--- /dev/null
+++ b/_posts/2020-05-12-how-to-create-your-own-string-interpolator.md
@@ -0,0 +1,137 @@
+---
+title: "How to Create Your Own Custom String Interpolator"
+date: 2020-05-12
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, tricks]
+excerpt: "Yep, you can write your own interpolator that looks like it was built into the Scala language. Learn how."
+---
+This article will show you a less-known customizable part of Scala that will allow you to build powerful tools that seem to be part of the language itself. The article is for intermediate to advanced Scala programmers who know how implicit classes work.
+
+## The Background
+
+You're surely well aware of the standard Scala string interpolators. They allow us to inject values and even whole expressions into a string.
+
+The best known and most used interpolator is the S interpolator, which simply expands values or expressions inside a string.
+
+```scala
+val lifeOfPi = 3.14159
+val sInterpolator = s"The value of pi is $lifeOfPi. Half of pi is ${lifeOfPi / 2}"
+```
+
+This interpolator will simply call the toString methods of every value and expression that we expand, and the results will be part of the resulting string
+
+```Perl
+The value of pi is 3.14159. Half of pi is 1.570795
+```
+
+Then we have the Raw interpolator, which is the same as the S interpolator, except that it doesn't escape characters, but keeps them exactly as they are:
+
+```scala
+val rawIterpolator = raw"The value of pi is $lifeOfPi\n <-- this is not a newline"
+```
+
+Normally, the \n would trigger a new line, but in a Raw interpolator, it doesn't:
+
+```Perl
+The value of pi is 3.14159\n <-- this is not a newline
+```
+
+And finally we have the F interpolator, which has the ability to control the format in which values are shown. It has similar functionality to standard printf, such as controlling the number of decimals in a number:
+
+```scala
+val fInterpolator = f"The approximate value of pi is $lifeOfPi%3.2f"
+```
+
+```Perl
+The approximate value of pi is 3.14
+```
+
+## The Motivation
+
+If you've worked with Scala libraries and tools, you might have noticed other expressions that look like interpolators. For example, Spark or Slick:
+
+```scala
+val myDataFrame = input.select($"col1", $"col2")
+```
+
+```scala
+val query = sql"Select * from citizens where ..."
+```
+
+These libraries make it seem that their interpolators are part of the language itself. We can also do that.
+
+## The Scenario
+
+I'm going to assume a simple scenario: imagine you are using the following case class A LOT in your library:
+
+```scala
+case class Person(name: String, age: Int)
+```
+
+and you are doing a lot of parsing from strings in the form of "name,age" into instances of this Person class:
+
+```scala
+def stringToPerson(line: String): Person = {
+ // assume the strings are always "name,age"
+ val tokens = line.split(",")
+ Person(tokens(0), tokens(1).toInt)
+}
+
+val bob = stringToPerson("Bob,55")
+// and you're calling stringToPerson everywhere
+```
+
+I'll show you how you can create an interpolator so you can write
+
+```scala
+val bob = person"Bob,55"
+```
+
+as if the "person" interpolator was baked into the language itself
+
+## The Mechanics
+
+A custom interpolator needs only two things: an implicit wrapper over a special class called StringContext, and a method whose name is identical to the name of the interpolator you want to create. For "person", the method name needs to be "person".
+
+```scala
+implicit class PersonInterpolator(sc: StringContext) {
+ def person(args: Any*): Person = {
+ // logic here
+ }
+}
+```
+
+The method "person" needs to take Any* as argument: these are all the expressions you can inject into a string. Let me explain. When you write
+
+```scala
+s"The value of pi is $lifeOfPi. Half of pi is ${lifeOfPi / 2}"
+```
+
+The values you expand with the dollar sign are called _arguments_, and can be of any type (hence the type Any), while the pieces of string in between the arguments are called _parts_ and you can access them by sc.parts. In the method "person", you have access to both, so you can process them as you see fit. I'm just going to concatenate them all, and parse the Person from the resulting String:
+
+```scala
+implicit class PersonInterpolator(sc: StringContext) {
+ def person(args: Any*): Person = {
+ // concatenate everything: use the built-in S method (which happens to be used in the S interpolator)
+ val tokens = sc.s(args: _*).split(",")
+ Person(tokens(0), tokens(1).toInt)
+ }
+}
+```
+
+And finally you will be able to do
+
+```scala
+val name = "Bob"
+val age = 23
+val bob = person"$name,$age"
+```
+
+which will (behind the scenes) invoke the "person" method from a new instance of PersonInterpolator created with the StringContext obtained by the compiler after parsing the string and isolating its "parts" and "arguments".
+
+Potential drawback: instantiation of the PersonInterpolator many times if you're doing lots of these conversions.
+
+## A Powerful tool
+
+Custom interpolation is a nice tool for making various functionalities in your library seem like part of the language. It's (usually) short and straightforward, while making user code also short and self-explanatory.
diff --git a/_posts/2020-05-21-scala-syntax-tricks-for-expressiveness.md b/_posts/2020-05-21-scala-syntax-tricks-for-expressiveness.md
new file mode 100644
index 000000000000..a34890913c0e
--- /dev/null
+++ b/_posts/2020-05-21-scala-syntax-tricks-for-expressiveness.md
@@ -0,0 +1,147 @@
+---
+title: "5 Nice Scala Tricks for Expressiveness"
+date: 2020-05-21
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, tricks]
+excerpt: "Scala is an amazingly expressive language, but even the most experienced devs aren't aware of all the subtleties."
+---
+This article is for Scala programmers of all levels who are at least familiar with the absolute essentials (values, classes, functions etc.). In this post I'm going to show you a few syntax tricks and sugar elements that make Scala such a powerful, expressive language. Many of these techniques are not so well-known, so there are probably at least some of these that you may not have seen before.
+
+## Trick 1 - the single abstract method pattern
+
+Since Scala 2.12, abstract classes or traits with a single unimplemented method can be reduced to lambdas. Here's an example:
+
+```scala
+trait Action {
+ def act(x: Int): Int
+}
+
+val myAction: Action = (x: Int) => x + 1
+```
+
+In this case, the compiler can automatically convert the lambda into an anonymous instance of Action, so it's as if I said
+
+```scala
+val action: Action = new Action {
+ override def act(x: Int) = x + 1
+}
+```
+
+This is particularly useful with spawning JVM Threads, because they traditionally take a Runnable - which is an interface with a single abstract method - as argument, so we can now create Threads very easily:
+
+```scala
+new Thread(() => println("I run easy!")).start()
+```
+
+## Trick 2 - left-associative methods
+
+Ever wondered how you could write `2 :: someList`? You're probably aware that the `::` method is used infix, but it's supposed to be a member of the List type, not on Int, so how does that work?
+
+The answer is baked into the Scala syntax: methods with non-alphanumeric names which end in a colon, like `::`, are _right-associative_. By this we mean that we can write
+
+```scala
+1 :: 2 :: 3 :: List()
+```
+
+which in fact means
+
+```scala
+1 :: (2 :: (3 :: List()))
+```
+
+and the compiler will rewrite it in the "standard" way as
+
+```scala
+List().::(3).::(2).::(1)
+```
+
+Can you write your own operator like that? Sure you can! Here's an example:
+
+```scala
+class MessageQueue[T] {
+ // an enqueue method
+ def -->:(value: T): MessageQueue[T]
+}
+
+val queue = 3 -->: 2 -->: 1 -->: new MessageQueue[Int]
+```
+
+## Trick 3 - baked-in "setters"
+
+If you come from Java, the getter/setter pattern is all too familiar. In Scala, we discourage mutable data structures in general, but in case we do want them, we don't want the fields exposed as vars. At the same time, the old getThis and setThis pattern is all too verbose.
+
+```scala
+class MutableIntWrapper {
+ private var internalValue = 0
+ // getter
+ def value = internalValue
+ // setter
+ def value_=(newValue: Int) = { internalValue = value }
+}
+```
+
+When we write something like that, we can now write a much more natural setter statement:
+
+```scala
+val wrapper = new MutableIntWrapper
+wrapper.value = 43 // same as wrapper.value_=(43)
+```
+
+## Trick 4 - multi-word members
+
+Scala allows multi-word method and field names, to more clearly express what they mean and avoid having to rename values which would otherwise contain restricted characters.
+
+```scala
+class Person(name: String) {
+ def `then said`(thing: String) = s"$name then said: $thing"
+}
+
+val jim = new Person("Jim")
+jim `then said` "Scala is pretty awesome!"
+```
+
+A real-life example where this kind of naming is used successfully is Akka HTTP, so that it can keep the familiar HTTP terms exactly as they are. Here's a real request I make to the server which does syntax highlighting for this blog:
+
+```scala
+val request = HttpRequest(
+ method = HttpMethods.POST,
+ uri = "http://markup.su/api/highlighter",
+ entity = HttpEntity(
+ ContentTypes.`application/x-www-form-urlencoded`, // <--- look here
+ s"source=${URLEncoder.encode(source, "UTF-8")}&language=Scala&theme=Sunburst"
+ )
+)
+```
+
+## Trick 5 - backtick pattern matching
+
+Another use of backticks is a small but powerful feature of pattern matching: the ability to match an existing variable exactly. Assume you have a value `meaningOfLife` in your code and you want to match it in a PM expression. If you write
+
+```scala
+val pm = data match {
+ case meaningOfLife => ...
+}
+```
+
+then all you're doing is shadowing your variable inside the PM case. You could do it like this, a bit awkwardly:
+
+```scala
+val pm = data match {
+ case m if m == meaningOfLife => ...
+}
+```
+
+but you can do this:
+
+```scala
+val pm = data match {
+ case `meaningOfLife` => ...
+}
+```
+
+which is a shorthand for saying "match the exact value this variable has now".
+
+## Power comes in small things
+
+Scala has some powerful and expressive features that allow one to write elegant and concise code. My bet is that even if you're an advanced Scala programmer, there is at least one trick above that you did not know about. I hope they will all be useful!
diff --git a/_posts/2020-06-01-monads.md b/_posts/2020-06-01-monads.md
new file mode 100644
index 000000000000..3c938ac95b07
--- /dev/null
+++ b/_posts/2020-06-01-monads.md
@@ -0,0 +1,284 @@
+---
+title: "A Monads Approach for Beginners, in Scala"
+date: 2020-06-01
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, functional programming]
+excerpt: "A Scala tutorial on Monads that makes no assumptions and starts from practical needs. We derive the monad patterns (\"laws\") from scratch."
+---
+There are so many tutorials on Monads it's not even funny. In fact, it's not funny at all - the more analogies people make, the more confused readers and listeners seem to be, because how could you bridge the gap between burritos, programmable semicolons, applicative functors? Perhaps you could, IF you already knew IT.
+
+And by IT, I mean the M word.
+
+As much as I like to talk by way of analogies, I believe monads are not well-served by them. I scoured the web trying to find commonalities that would stick, and I couldn't. Instead, one common question I kept seeing in comments was: OK, so how are these things useful? So in this (probably long) article I'll address them by the invariable question in a curious programmer's mind: _why are they useful to ME?_
+
+Unlike burritos and abstract math, the goal of this article is to write code; we will show how monads solve programming problems, how they make code easier to understand and how they make us happy and more productive.
+
+The idea of a monad is not so difficult but it's notoriously hard to wrap your head around because it's so general. Once it clicks, it feels simple and profound, which probably explains the itch for the gazillions of tutorials on the subject. My hope is that after this article, you too can go "hm, that wasn't so hard".
+
+## Part 1: the pattern
+
+Monads are types which take values and do something interesting to them by following a certain structure. I spent 5 minutes writing that sentence. 99% of the value of monads is in their structure, which I hope will become part of your mental model. So in what follows we'll focus on the structure.
+
+Assume for example, I want to define a class which prevents multi-threaded access to a value:
+
+```scala
+case class SafeValue[+T](private val internalValue: T) {
+ def get: T = synchronized {
+ // note: this is just a simplified example and not 100% foolproof
+ // thread safety is a very complicated topic and I don't want to digress there
+ // replace the logic here with code that might be interesting for any reason
+ internalValue
+ }
+}
+```
+
+Also assume some external API gave me one of those SafeValues and I want to transform it into another SafeValue:
+
+```scala
+val safeString: SafeValue[String] = gimmeSafeValue("Scala is awesome") // obtained from elsewhere
+val string = safeString.get
+val upperString = string.toUpperCase()
+val upperSafeString = SafeValue(upperString)
+```
+
+Pay close attention to this pattern. Someone gives you a safe value, you unwrap it, transform it, then create another safe value.
+
+Instead of dealing with things "sequentially" in this way, we can define the transformation in the safe value class itself:
+
+```scala
+// more elegant (and safe): use a function to transform the element, inside the wrapper
+case class SafeWrapper2[+T](private val internalValue: T) {
+ def transform[S](transformer: T => SafeWrapper2[S]): SafeWrapper2[S] = synchronized {
+ transformer(internalValue)
+ }
+}
+```
+
+In this way, we can simply do
+
+```scala
+val safeString2 = SafeWrapper2("Scala is awesome")
+val upperSafeString2 = safeString2.transform(string => SafeWrapper2(string.toUpperCase()))
+```
+
+This is more concise, more elegant, much safer, and more OO-encapsulated, because you never need to get the internal value of the wrapper.
+
+This pattern (and its refactor) is very common, and is composed of two things:
+
+ 1. the ability to wrap a value into my (more interesting) type - in OO terms this is just a "constructor"; we call this _unit_, or _pure_, or _apply_
+ 2. a function that transforms a wrapper into another wrapper (perhaps of another type) in the same style as the above - we usually call this _bind_ or _flatMap_
+
+When you're dealing with this extract-transform-wrap pattern (I'll call this ETW), you've created the conditions for a monad.
+
+## Part 2: examples from the jungle
+### Example 1: A census application
+Assume you're populating a database with people, and you have the following code:
+
+```scala
+case class Person(firstName: String, lastName: String) {
+ // you have a requirement that these fields must not be nulls
+ assert(firstName != null && lastName != null)
+}
+
+def getPerson(firstName: String, lastName: String): Person =
+ if (firstName != null) {
+ if (lastName != null) {
+ Person(firstName.capitalize, lastName.capitalize)
+ } else {
+ null
+ }
+ } else {
+ null
+ }
+```
+
+This is a defensive Java-style implementation, which is hard to read, hard to debug and very easy to nest further. Notice the pattern: "extract" the first name, transform it, "extract" the last name, transform it, then wrap them into something interesting.
+
+We can more elegantly solve this with Options. You've probably seen them before: they're like lists, except they have at most one element. Options have both structural elements of monads: a "constructor" and a flatMap. We can therefore simplify this code:
+
+```scala
+def getPersonBetter(firstName: String, lastName: String): Option[Person] =
+ Option(firstName).flatMap { fName =>
+ Option(lastName).flatMap { lName =>
+ Option(Person(fName.capitalize, lName.capitalize))
+ }
+ }
+```
+
+and we can then reduce this to for-comprehensions:
+
+```scala
+def getPersonFor(firstName: String, lastName: String): Option[Person] = for {
+ fName <- Option(firstName)
+ lName <- Option(lastName)
+} yield Person(fName.capitalize, lName.capitalize)
+```
+
+Notice that it's now much easier to read this code, because you have no more reason to do the null-checking logic in your head, you just leave to the Options. The null-checking part is the "interesting" thing they do.
+
+### Example 2: asynchronous fetches
+Imagine you're calling some async services for your online store, like fetching from some external resource:
+
+```scala
+case class User(id: String)
+case class Product(sku: String, price: Double)
+// ^ never use Double for currency IN YOUR LIFE
+
+def getUser(url: String): Future[User] = Future { ... }
+def getLastOrder(userId: String): Future[Product] = Future { ... }
+```
+
+and assume I want to get Daniel's last order from the store so I can send him a VAT invoice. However, all I know is the URL I need to call:
+
+```scala
+val userFuture = getUser("my.store.com/users/daniel")
+```
+
+Now what?
+
+Beginners usually get stuck here: "how do I wait for this Future to end so I can get its value and move on with my life?". So they onComplete the crap out of it:
+
+```scala
+userFuture.onComplete {
+ case Success(User(id)) =>
+ val lastOrder = getLastOrder(id)
+ lastOrder.onComplete {
+ case Success(Product(_, p)) =>
+ val vatIncludedPrice = p * 1.19
+ // then pass it on
+ }
+}
+```
+
+This is really bad, nested, hard to read and doesn't cover all cases anyway. Notice you're following the same ETW pattern: trying to extract an object, doing something with it and then wrap it back. Enter flatMaps again:
+
+```scala
+val vatIncludedPrice = getUser("my.store.com/users/daniel")
+ .flatMap(user => getLastOrder(user.id)) // relevant monad bit
+ .map(_.price * 1.19) // then do your thing
+```
+
+which you can then use to pass on to your logic computing VATs. Or even better:
+
+```scala
+val vatIncludedPriceFor = for {
+ user <- getUser("my.store.com/users/daniel")
+ order <- getLastOrder(user.id)
+} yield order.price * 1.19
+```
+
+### Example 3: double-for "loops"
+
+Lists follow the same pattern. Here's an example: you want to return a "checkerboard" (i.e. a cartesian product) of all possible combinations of elements from two lists.
+
+```scala
+val numbers = List(1,2,3)
+val chars = List('a', 'b', 'c')
+```
+
+Same pattern: get a value from numbers, get a value from chars, build a tuple from them, "repeat". Solvable with flatMaps:
+
+```scala
+val checkerboard = numbers.flatMap(number => chars.map(char => (number, char)))
+```
+
+Or better:
+
+```scala
+val checkerboard2 = for {
+ n <- numbers
+ c <- chars
+} yield (n, c)
+```
+
+which looks oddly similar to Java or imperative "loops". The reason why this looks similar is that monads inherently describe sequential ETW computations: extract this, then transform, then wrap. If you want to process it further, same thing: extract, transform, wrap.
+
+If you got this far, congratulations! The gist is this: whenever you need to ETW, you probably need a monad, which needs 1) a constructor, 2) a flatMap in the style of the above.
+
+## Part 3: the annoying axioms
+
+By reading the comments of various monad tutorials, I noticed that people are usually pushed away by the abstraction caused by math. I hope the examples above set the stage for some inherent properties of monads that I hope will prove self-evident. Here goes:
+
+### Property 1: left-identity, a.k.a. the "if I had it" pattern
+
+The ETW pattern is described in a single line: `MyThing(x).flatMap(f)`. You can express that in two ways:
+
+1. If you have the x the monad was built with, you want to apply the transformation on it, so you get `f(x)`.
+2. If you don't, you need the ETW pattern, so you need `MyThing(x).flatMap(f)`.
+
+Either way, you get the same thing. For all monads, `MyThing(x).flatMap(f) == f(x)`.
+
+### Property 2: left-associativity, a.k.a. the useless wrap
+
+Assume you have MySuperMonad(x). What should happen when you say
+
+```scala
+MySuperMonad(x).flatMap(x => MySuperMonad(x))
+```
+
+If it's not obvious yet, make it concrete:
+ - example with lists: `List(x).flatMap(x => List(x)) = ?`
+ - example with Futures: `Future(x).flatMap(x => Future(x))= ? `
+
+Nothing changes, right? An inherent property of monads is that `MySuperMonad(x).flatMap(x => MySuperMonad(x))` is the same as `MySuperMonad(x)`.
+
+### Property 3: right-associativity, a.k.a. ETW-ETW
+
+This one is critical, and describes the correctness of multiple flatMap chains on monads. Say you have a list and some transformations:
+
+```scala
+val numbers = List(1,2,3)
+val incrementer = (x: Int) => List(x, x + 1)
+val doubler = (x: Int) => List(x, 2 * x)
+```
+
+If you apply both transformations in flatMaps, you get:
+
+```scala
+[1,2,3].flatMap(incrementer).flatMap(doubler) =
+[1,2,2,3,3,4].flatMap(doubler) =
+[1,2,2,4,2,4,3,6,3,6,4,8]
+```
+
+Now, if you look at that list like this,
+
+```scala
+[1,2,2,4, 2,4,3,6, 3,6,4,8]
+```
+
+you can see which numbers were generated by each initial element of the list. In other words, we have
+
+```scala
+[
+ incrementer(1).flatMap(doubler),
+ incrementer(2).flatMap(doubler),
+ incrementer(3).flatMap(doubler)
+]
+```
+
+just as if a fictitious function `x => incrementer(x).flatMap(doubler)` was applied to each element. In other words,
+
+```scala
+numbers.flatMap(incrementer).flatMap(doubler) == numbers.flatMap(x => incrementer(x).flatMap(doubler))
+```
+
+Replace numbers with any monad and put any functions inside, and you get the boo-hoo-hoo scary axiom:
+
+```scala
+MyMonad(x).flatMap(f).flatMap(g) == MyMonad(x).flatMap(x => f(x).flatMap(g))
+```
+
+The pattern goes like this:
+ - extract
+ - transform & wrap
+ - extract again
+ - transform & wrap
+
+which is what the two flatMaps do; in the right hand side we're compressing the last 3 steps:
+ - extract
+ - transform & wrap & ETW
+
+## Epilogue
+
+I hope this dive into monads was as down-to-earth as possible. We use monads everywhere, often without noticing. Monads are just a mathematical label attached to something we otherwise use constantly, and the axioms look quite scary and bland. However, once you understand what monads want to solve, you naturally get a deep insight into the very nature of computation, and that alone is worth the effort of this tutorial and many before (and after) it.
diff --git a/_posts/2020-06-04-akka-http-to-heroku-in-x-minutes.md b/_posts/2020-06-04-akka-http-to-heroku-in-x-minutes.md
new file mode 100644
index 000000000000..d41393a5245e
--- /dev/null
+++ b/_posts/2020-06-04-akka-http-to-heroku-in-x-minutes.md
@@ -0,0 +1,160 @@
+---
+title: "Akka HTTP to Heroku in 10 Minutes"
+date: 2020-06-04
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka http, heroku, how to]
+excerpt: "Akka HTTP is super easy to deploy to a web server. Learn how to use Heroku and deploy your first Akka HTTP service in minutes."
+---
+This article is for newbies to Akka HTTP and for those who have written some Akka HTTP but have never deployed their own server. Here I'll teach you what you need to know and do so that you have your first server up and running on Heroku in just a few minutes.
+
+I'll be working in IntelliJ IDEA which creates an SBT project structure quickly, but you can also use the `sbt` command line to do the same things I do here. So here goes:
+
+## Step 1 - Intro
+
+We'll start by creating a vanilla Scala-SBT project in intelliJ:
+
+
+
+
+
+And after you click Finish, IntelliJ will create the appropriate project structure for you. Go to the `src/` folder, and create a package and then an application under it. I created a package called `server` and under it I started a simple `MyServer.scala`:
+
+```scala
+object MyServer {
+ def main(args: Array[String]): Unit = {
+ // will write in due time
+ }
+}
+```
+
+Once you've done that, it's time to "register" it as the runnable application so Heroku can identify it.
+
+## Step 2 - The Boilerplate
+
+IntelliJ will create a `project/` folder, where you will need to add a file called `plugins.sbt` and add an SBT plugin so you can package your application at the time you will deploy it to Heroku:
+
+```scala
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.12")
+```
+
+Then under your `build.sbt` you'll need to install the Akka libraries, refer to the packaging plugin and register the main class. Here is the entire content of the file:
+
+```scala
+name := "akka-http-test"
+
+version := "0.1"
+
+scalaVersion := "2.12.4"
+
+// this will add the ability to "stage" which is required for Heroku
+enablePlugins(JavaAppPackaging)
+
+// this specifies which class is the main class in the package
+mainClass in Compile := Some("server.MyServer")
+
+val akkaVersion = "2.6.5"
+val akkaHttpVersion = "10.1.12"
+
+// add the Akka HTTP libraries
+libraryDependencies ++= Seq(
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion,
+ "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion,
+ "com.typesafe.akka" %% "akka-http" % akkaHttpVersion
+)
+```
+
+After that, IntelliJ will usually pop up a small dialog asking you to "Import Changes". Click that and wait for a minute to download and install the libraries and plugin. Otherwise, go to View -> Tool Windows -> SBT and click on the tiny refresh icon to do the same thing.
+
+After you don't see any red in build.sbt, you're good to go and you can write the actual code of your server
+
+## Step 3 - The Fun Part
+
+It's probably the place we care about the most: the actual writing of the logic that will be our server. Akka HTTP is a huge set of libraries and with lots of capabilities. In this article I'll focus on a server that when hit with a GET request, will respond with a small HTML which will be seen in your browser. So here goes. Go to `MyServer.scala` and add any HTML you fancy:
+
+```scala
+ val content =
+ """
+ |
+ |
+ |
+ | This is an HTML page served by Akka HTTP!
+ |
+ |
+ """
+```
+
+Then you will define the logic of the server by a Directive, which is a fancy DSL to compose routing mechanisms very quickly:
+
+```scala
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.model.{ContentTypes, HttpEntity}
+```
+
+and inside your server application:
+
+```scala
+val route = get {
+ complete(
+ HttpEntity(
+ ContentTypes.`text/html(UTF-8)`,
+ content
+ )
+ )
+}
+```
+
+which means that this server will respond to any GET request with an HTTP response with the content type `text/html(UTF-8)`, and with the payload given by the content (HTML string) you defined earlier).
+
+Then in the main method, you need to actually bind this routing logic on a hostname/port combination. On Heroku, you can't bind to localhost, so you will bind to `0.0.0.0`. Here's how the main method looks like:
+
+```scala
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+
+def main(args: Array[String]): Unit = {
+ // Akka HTTP needs an actor system to run
+ implicit val system = ActorSystem("Server")
+
+ // set hostname/port combination
+ val host = "0.0.0.0"
+ val port: Int = sys.env.getOrElse("PORT", "8080").toInt
+
+ // this actually starts the server
+ Http().bindAndHandle(route, host, port)
+}
+```
+
+And that's it! Your application is now ready. All you have to do is deploy it to Heroku.
+
+## Step 4 - The Deploy
+
+Do the steps below, they will take just a couple of minutes:
+
+1. If you don't have a Heroku account, create one .
+2. Then, create a new app , give it a name.
+3. After that, make sure you install the Heroku command line .
+
+After you have the Heroku command line, open a terminal and go to the directory of your Akka HTTP project, and run the following: first, make your project a Git repository:
+
+```scala
+$ git init
+$ git add .
+$ git commit -m "my first server"
+```
+
+Then, login to Heroku and make Heroku detect this project as a Scala application:
+
+```scala
+$ heroku login
+```
+
+which will also add the `heroku` remote for this project. After that, all you have to do is
+
+```scala
+$ git push heroku master
+```
+
+## Part 5 - Enjoy
+
+Navigate to https://replace-this-with-your-akka-http-server.herokuapp.com and see your first Akka HTTP server in action!
diff --git a/_posts/2020-06-11-why-are-typeclasses-useful.md b/_posts/2020-06-11-why-are-typeclasses-useful.md
new file mode 100644
index 000000000000..483a003db576
--- /dev/null
+++ b/_posts/2020-06-11-why-are-typeclasses-useful.md
@@ -0,0 +1,78 @@
+---
+title: "Why are Type Classes Useful in Scala?"
+date: 2020-06-11
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, functional programming, type system]
+excerpt: "FP fans talk about pure functional programming in Scala with type classes. They're hard. Why do we really need them?"
+---
+This article is for the Scala programmer who knows what implicits are and (for the most part) how they work. This article will also involve a somewhat high degree of abstraction.
+
+Type classes are these super-abstract concepts in functional programming, which FP purists and Haskellers eat for breakfast, lunch and dinner. Wikipedia says type classes are "type system constructs that support ad hoc polymorphism". What the heck does that mean?
+
+This article wants to be down to earth.
+
+## The problem
+
+Ever since generics were invented, you've surely come across the need for specialized implementations. In other words, let's say you have a method
+
+```scala
+def processMyList[T](list: List[T]): T = {
+ // aggregate a list
+}
+```
+
+and in this function's implementation, you need a result that is obtained by processing the list argument - for the sake of example, let's say you "sum" all the elements of the list. But here's the twist: if the list is a list of integers, the "sum" should be the real sum of the elements; if the list contains strings, then the "sum" should be the concatenation of all the elements. For all other types, we should not be able to use this method. And we want to do everything automatically.
+
+If you're in Java, you can kiss this dream goodbye. If you're in C++, you have to resort to template specializations. In Scala, we have an elegant way of dealing with it.
+
+## Plugging an implicit
+
+In Scala, we can enhance this method with implicit arguments which can enhance its capability and constrain its use at the same time. Let me give an example.
+
+```scala
+trait Summable[T] {
+ def sumElements(list: List[T]): T
+}
+```
+
+This is a trait that defines the capability of aggregating a list into a single element. We can of course add some implementations for Int and String, the way we like it:
+
+```scala
+implicit object IntSummable extends Summable[Int] {
+ def sumElements(list: List[Int]): Int = list.sum
+}
+
+implicit object StringSummable extends Summable[String] {
+ def sumElements(list: List[String]): String = list.mkString("")
+}
+```
+
+As you can see, two very different implementations of the "sum" we can perform. We can then enhance the original method like this:
+
+```scala
+def processMyList[T](list: List[T])(implicit summable: Summable[T]): T =
+ summable.sumElements(list)
+```
+
+If you try this, you will notice that it works for Strings and Ints, and it doesn't even compile for anything else:
+
+```scala
+processMyList(List(1,2,3)) // 6
+processMyList(List("Scala ", "is ", "awesome")) // "Scala is awesome"
+processMyList(List(true, true, false)) // ERROR
+```
+
+In this way, the implicit works as both a capability enhancer and a type constraint, because if the compiler cannot find an implicit instance of a ListAggregation of that particular type, i.e. your specialized implementation, then it's certain that the code can't run.
+
+## The fancy terms
+
+Did you hear "type class" anywhere? You don't need to. If you absolutely must hear the terms, let me break them down:
+
+The behavior we've just implemented is called "ad hoc polymorphism" because the `sumElements` ability is unlocked only in the presence of an implicit instance of the trait which provides the method definition, right there when it's called, hence the "ad hoc" name. "Polymorphism" because the implementations we can provide can obviously be different for different types, as we did with Int and String.
+
+The trait `Summable[T]` itself is nothing special. However, when you combine it with one/more implicit instances of the trait - and in our case we `IntSummable` and `StringSummable` - we have a pattern, which we generally call a "type class". This structure allows us to define specific implementations for certain types and not for others, in the "ad hoc polymorphic" style we did earlier.
+
+## Capish?
+
+I hope this article cut through the abstraction weeds and was as down to earth as possible. Dying for feedback, so leave your comments here!
diff --git a/_posts/2020-06-17-kafka-akka-spark.md b/_posts/2020-06-17-kafka-akka-spark.md
new file mode 100644
index 000000000000..93b64f2b7d2a
--- /dev/null
+++ b/_posts/2020-06-17-kafka-akka-spark.md
@@ -0,0 +1,290 @@
+---
+title: "Comparing Akka Streams, Kafka Streams and Spark Streaming"
+date: 2020-06-17
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kafka, akka streams, spark, spark streaming]
+excerpt: "We compare three of the most popular streaming libraries and see where Akka Streams, Kafka Streams and Spark Streaming are most useful."
+---
+This article is for the Java/Scala programmer who wants to decide which framework to use for the streaming part of a massive application, or simply wants to know the fundamental differences between them, just in case. I'm going to write Scala, but all the frameworks I'm going to describe also have Java APIs.
+
+I'm going to discuss the main strengths and weaknesses of _Akka Streams, Kafka Streams and Spark Streaming_, and I'm going to give you a feel of how you would use them in a very simple word-counting application, which is one of the basic things to start with when one learns any distributed programming tool.
+
+## Kafka Streams
+
+Kafka Streams is a client library for processing unbounded data . What does that mean? Client library means that the application we write uses the services provided by another infrastructure (this case a Kafka cluster). So we interact with a cluster to process a potentially endless stream of data. The data is represented as key-value records, which makes them easy to identify, and they are organized into topics, which are durable event logs, essentially persistent queues of data which is written to disk and replicated. In this architecture, we have producer applications pushing records into these topics - e.g. if you have an online store, you'd like to keep track of everything that happened to an order - and on the other hand we have multiple consumer applications which read the data in various ways and starting at various points in time in these topics.
+
+This way of structuring the data allows for highly distributed and scalable architectures, which are also fault-tolerant. Kafka also embeds the exactly-once messaging semantics, which means that if you send a record to Kafka, you will be sure that it gets to the cluster and it's written once with no duplicates. This is particularly important because this mechanism is extremely hard to obtain in distributed systems in general.
+
+From the way Kafka is organized, the API allows a Java or Scala application to interact with a Kafka cluster independently of other applications that might be using it at the same time. So this independence of applications accessing the same distributed and scalable service naturally incentivizes the use of independent microservices in your big application.
+
+## How Kafka Streams Looks Like
+
+```scala
+object WordCountApplication extends App {
+ import Serdes._
+ val props: Properties = {
+ val p = new Properties()
+ p.put(StreamsConfig.APPLICATION_ID_CONFIG, "myFabulousWordCount")
+ p.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "my-kafka-broker-url:9092")
+ p
+ }
+
+ val builder: StreamsBuilder = new StreamsBuilder
+ val textLines: KStream[String, String] =
+ builder.stream[String, String]("TextLinesTopic")
+ val wordCounts: KTable[String, Long] = textLines
+ .flatMapValues(textLine => textLine.toLowerCase.split("\\W+"))
+ .groupBy((_, word) => word)
+ .count()(Materialized.as("word-counts-table"))
+
+ wordCounts.toStream.to("WordsWithCountsTopic")
+ val streams: KafkaStreams = new KafkaStreams(builder.build(), props)
+ streams.start()
+
+ sys.ShutdownHookThread {
+ streams.close(10, TimeUnit.SECONDS)
+ }
+}
+```
+
+This is how a word count application would look like in Kafka Streams. Now this code is quite heavy to take all in at once, so I'll break it down.
+
+```scala
+ import Serdes._
+```
+
+Kafka stores records in binary for performance, which means it's up to us to serialize and deserialize them. We can do this with this import of serializers and deserializers (Serdes) automatically in Scala.
+
+```scala
+ val props: Properties = {
+ val p = new Properties()
+ p.put(StreamsConfig.APPLICATION_ID_CONFIG, "myFabulousWordCount")
+ p.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "my-kafka-broker-url:9092")
+ p
+ }
+```
+
+The first part of an application invariably needs to configure the details of the Kafka cluster it's going to connect to. This will use a Java-style API, which I personally hate as a predominantly Scala programmer. Java folks might be much more comfortable with this.
+
+```scala
+ val builder: StreamsBuilder = new StreamsBuilder
+ val textLines: KStream[String, String] =
+ builder.stream[String, String]("TextLinesTopic")
+```
+
+Next, we read the records as key-value pairs from the topic that we want, using a builder pattern.
+
+```scala
+ val wordCounts: KTable[String, Long] = textLines
+ .flatMapValues(textLine => textLine.toLowerCase.split("\\W+"))
+ .groupBy((_, word) => word)
+ .count()(Materialized.as("word-counts-table"))
+```
+
+Then we will apply some functional programming operators on the stream, as if it were a collection, and we will turn that stream into a table. Kafka Streams has this notion of a table which allows for data aggregation and processing. Kafka has a stream-table duality which allows us to convert back and forth between them.
+
+```scala
+ wordCounts.toStream.to("WordsWithCountsTopic")
+```
+
+Speaking of conversion, we might want to convert this table to a stream and feed it into another topic that some other application might be interested in reading from.
+
+```scala
+ val streams: KafkaStreams = new KafkaStreams(builder.build(), props)
+ streams.start()
+
+ sys.ShutdownHookThread {
+ streams.close(10, TimeUnit.SECONDS)
+ }
+```
+
+And finally, we just need to start the streams and setup a graceful stop, because otherwise the streams are static and won't do anything.
+
+## Kafka Streams Strengths and Weaknesses
+
+The major benefit of Kafka Streams is that a Kafka cluster will give you _high speed, fault tolerance and high scalability_. Kafka also provides this _exactly-once message sending_ semantics, which is really hard in distributed systems as otherwise many other frameworks are unable to offer this kind of guarantee and as such you might end up with either duplicates or data loss. At the same time, Kafka _incentivizes the use of microservices_ using the same message bus to communicate, and so you have the power and control to set up your own inter-microservice communication protocol via Kafka.
+
+Of course, Kafka is not without its downsides. As a predominantly Scala programmer, I hate Kafka's _imperative Java-style API_ but I'll swallow the pain for Kafka's otherwise excellent capabilities. Another downside is that if you want to have Kafka in your architecture, you will need to set up _a separate Kafka cluster_ that you will need to manage, even though you won't necessarily need to allocate dedicated machines. Kafka is also highly configurable, but you need to know the configurations in advance to make it work at all. Finally, Kafka supports only the producer-consumer type architectures. You can also simulate other architectures, but it never feels as though it was meant to support it naturally - or, as some people like to say, "first-class" - so even though Kafka is excellent for this particular purpose, it's not as versatile as other frameworks.
+
+## Akka Streams
+
+That said, let's move onto Akka Streams. Akka Streams is an extremely high-performance library built for the JVM, written in Scala, and it's the canonical implementation of the Reactive Streams specification. The tenets of the Reactive Manifesto are _responsiveness, elasticity, fault-tolerance and message-driven semantics_, all of which are at the core of Akka Streams. In here you have total control over processing individual records in what could be an infinite amount of data and 100% control over streaming topologies of any configuration. Akka Streams is powered by the very successful actor model of concurrency and streaming components are built off asynchronous individual components that can process the data in any way you want.
+
+The major strengths of Akka Streams are again _high scalability and fault tolerance_, but in a different way, as we will see in the code. Akka Streams offers an _extremely versatile and concise streaming API_ which evolved into its own Scala-based DSL, and you can simply "plug in" components and just start them. At the same time, Akka Streams also offers a low-level GraphStage API which gives you a _high degree of control_ over the individual logic of particular components.
+
+As I mentioned, Akka Streams is highly performant and fault-tolerant, but it was built for a different purpose. While in Kafka you used it as a message bus and your application was a client API for the Kafka cluster, in here Akka Streams is _an integral part_ of your application's logic. You can imagine Akka Streams like the circulatory system of your application, whereas Kafka is just an external well-organized blood reservoir.
+
+## How Akka Streams Looks Like
+
+```scala
+val source1 = Source(List("Akka", "is", "awesome"))
+val source2 = Source(List("learning", "Akka", "Streams"))
+val sink = Sink.foreach[(String, Int)](println)
+
+val graph = GraphDSL.create() { implicit builder =>
+ import GraphDSL.Implicits._
+
+ val wordCounter = Flow[String]
+ .fold[Map[String, Int]](Map()) { (map, record) =>
+ map + (record -> (map.getOrElse(record, 0) + 1))
+ }
+ .flatMapConcat(m => Source(m.toList))
+
+ val merge = builder.add(Merge[String](2))
+ val counter = builder.add(wordCounter)
+
+ source1 ~> merge ~> counter ~> sink
+ source2 ~> merge
+
+ ClosedShape
+}
+
+RunnableGraph.fromGraph(graph).run()
+```
+
+So let's look at how we could build a word count application with Akka Streams. Even if you're experienced with Scala, this code might still be too concise. This is one of the drawbacks of Akka Streams. From my experience and from my students' experience, it's generally pretty tough on beginners regardless of your previous experience. But fear not, I'll break it down. Here are the main pieces of the code.
+
+```scala
+val source1 = Source(List("Akka", "is", "awesome"))
+val source2 = Source(List("learning", "Akka", "Streams"))
+val sink = Sink.foreach[(String, Int)](println)
+```
+
+The first 3 lines build the original sources, which will emit the elements (in our case strings) asynchronously.
+
+```scala
+ val wordCounter = Flow[String]
+ .fold[Map[String, Int]](Map()) { (map, record) =>
+ map + (record -> (map.getOrElse(record, 0) + 1))
+ }
+ .flatMapConcat(m => Source(m.toList))
+```
+
+The interesting piece which actually computes the word count is here, where we do a fold like we would on a simple list of Strings. Looks very concise, hard to look at and it definitely needs some getting used to, but if you've worked with Scala collections a lot, this shouldn't look too alien. I hope.
+
+```scala
+val merge = builder.add(Merge[String](2))
+val counter = builder.add(wordCounter)
+
+source1 ~> merge ~> counter ~> sink
+source2 ~> merge
+```
+
+
+However, that's not the interesting piece and the big strength of Akka Streams. The magic happens here, where we simply plug the different streaming components with their own logic.
+
+The visual graph that resembles the stream looks like this.
+
+
+
+Now take a look at the code.
+
+```scala
+source1 ~> merge ~> counter ~> sink
+source2 ~> merge
+```
+
+So notice we have a very similar representation of the streaming topology directly in the code! Some of you might go "meh", but it's hard to overstate how easy it is to construct arbitrary streaming layouts in just 3 lines of code, completely asynchronous, high speed and fault-tolerant.
+
+## Akka Streams Strengths and Weaknesses
+
+So let me discuss the big ups and downs with Akka Streams.
+
+Because Akka Streams is a Reactive Streams implementation, it naturally follows all the tenets of the Reactive Manifesto, which are _elasticity, responsiveness, fault-tolerance and message-driven behavior_. As such, it offers extreme speeds and high scalability. _The API is extremely powerful_ and is the closest I've ever seen to a fully visual representation. At the same time, Akka Streams offers a low-level GraphStage API which gives you all the control in the world for _custom streaming logic_, for example to batch data or to manually interrupt or redirect a stream – really, the possibilities are endless. One of the best parts of Akka Streams is that _it can seamlessly connect to Kafka_ with the Alpakka Kafka connector . Akka Streams was built as a development library for applications, and so you're not programming as a client API, like you did with Kafka, but you're using it as you would any other library for building a distributed application.
+
+The downside of Akka Streams are that Akka Streams is _very hard to learn and wrap your head around_. I know people who were joking that the people at Lightbend made it so hard so that only they could understand it. At the same time, Akka Streams is like the C++ of streaming, _you can easily shoot yourself in the foot_ even though you have all the power in the world (perhaps especially because you have all the power in the world). At the same time, although Akka (in all of its might) can scale really well if you use the entire suite with clustering & co, integrating Akka Streams for scalability will be a challenge in general. The fact that Akka Streams is _an integral part of your application_ is both a blessing and a curse, because like any "building" library, you need to adopt a certain style of thinking.
+
+## Spark Streaming
+
+Now let's move on to Spark Streaming, which is a natural streaming extension of the massively popular Spark distributed computing engine. The purpose of Spark streaming is to process endless big data at scale. You have a choice between _two API levels_: one low-level high-control API with Discretized Streams (DStreams) and the other very familiar DataFrame API, which is also called Structured Streaming and offers an identical API to the regular "static" big data. Spark is _naturally scalable and fault-tolerant_ from the get-go and functions in two output modes: micro-batch, in which at every interval Spark will add up all the data it gathered so far, and continuous mode which offers lower-latency (experimental at the moment of this writing).
+
+The big strengths of Spark are the capacity to deal with _massive data_, a familiar _DataFrame or SQL API_ and the rich _Spark UI_ which allows you to monitor and track the performance of your jobs in real time.
+
+It's worth noting that Spark will need _a dedicated compute cluster_ to run, which is usually costly in production. At the same time, Spark is extremely _configurable_, and if you know how to tune Spark properly, you'll get some giant performance improvements.
+
+## How Spark Streaming Looks Like
+
+```scala
+val spark = SparkSession.builder()
+ .appName("Word count")
+ .master("local[*]")
+ .getOrCreate()
+
+ val streamingDF = spark.readStream
+ .format("kafka")
+ .option("kafka.bootstrap.servers", "your-kafka-broker:9092")
+ .option("subscribe", "myTopic")
+ .load()
+
+ val wordCount = streamingDF
+ .selectExpr("cast(value as string) as word")
+ .groupBy("word")
+ .count()
+
+ wordCount.writeStream
+ .format("console")
+ .outputMode("append")
+ .start()
+ .awaitTermination()
+```
+
+So let's go though the standard word counting application and in here we will use the high-level Structured Streaming API. Clean and separable. Let's break it down
+
+```scala
+val spark = SparkSession.builder()
+ .appName("Word count")
+ .master("local[*]")
+ .getOrCreate()
+```
+
+The only boilerplate you'll need is to start a Spark Session. After that...
+
+```scala
+val streamingDF = spark.readStream
+ .format("kafka")
+ .option("kafka.bootstrap.servers", "your-kafka-broker:9092")
+ .option("subscribe", "myTopic")
+ .load()
+```
+
+...you can read your data by specifying the data source to read from, and Spark Streaming naturally supports Kafka out of the box.
+
+```scala
+val wordCount = streamingDF
+ .selectExpr("cast(value as string) as word")
+ .groupBy("word")
+ .count()
+```
+
+The actual logic is also plain and simple. In SQL, that's just a "group by" with a count, which we are doing here. Because Kafka stores data in binary, we have to add a cast at the beginning.
+
+```scala
+wordCount.writeStream
+ .format("console")
+ .outputMode("append")
+ .start()
+ .awaitTermination()
+```
+
+Finally, all you have to do is to point the stream to an output sink (where again we can use Kafka) and just start the streaming query.
+
+## Spark Streaming Strengths and Weaknesses
+
+So let's discuss the ups and downs with Spark Streaming.
+
+Spark Streaming was built for _big data_ from day 1. It offers _fault-tolerance_ as the original Spark core and _two APIs_ for dealing with data: one low-level high-control with DStreams and one hands-off with a familiar structure in the form of DataFrames and SQL. One nice feature of Spark is the ability to deal with late data based on event time and _watermarks_, which is very powerful in real life. Spark is highly configurable with massive perf benefits if used right and can _connect to Kafka_ via its built-in connector either as data input or data output. Not least, Spark also benefits from a massive community with excellent documentation and help. As an added benefit, Spark can also be quickly spun up locally for smaller data processing.
+
+As with the other frameworks, Spark is not perfect, though. The DataFrame and SQL APIs are cushy and familiar, but as a functional programmer I get a small stomach squeeze because _some type safety is lost_ at compile time. Of course, you have Datasets, but then you lose some performance if you pour in lambdas. Spark Streaming is very good for big data and micro-batch processing, but it's currently _not stellar at low-latency_ race unless you use the continuous mode which offers few guarantees and is experimental as we speak. Lastly, Spark _will need a dedicated cluster_ to run, so depending on your needs, you might be forced to spend a little more on compute if you go down this route.
+
+## What to use When
+
+Now the final piece: when should you use what? Naturally, every framework was built with a certain intent and we'll lay them here.
+
+Akka Streams is best for high-performance systems, _when you want to bake Akka Streams into your application_. It has an extremely powerful API, but unless you know what you're doing, it might be easy to shoot yourself (accidentally or not).
+
+Kafka on the other hand works best as an external high performance message bus for your applications, so if you want _microservices to read and write to and from a common event store_, you might be best with Kafka. However, its Java-style API is cumbersome, but I understand it might be a matter of code cleanliness and taste.
+
+Finally, Spark Streaming is without a doubt best for _big data computation_. Spark has always been good at that and here we also removed data bounds. However, at the moment of this recording, Spark Streaming is bad for actual application logic and low-latency, so you might want to use it as a _data aggregator_ to gather insights from your data.
+
+As a bonus, _everyone works with Kafka_ so if you want to add Kafka to your party, you can do that no matter which tool you pick.
diff --git a/_posts/2020-07-12-akka-streams-graphs.md b/_posts/2020-07-12-akka-streams-graphs.md
new file mode 100644
index 000000000000..125c26cb20cf
--- /dev/null
+++ b/_posts/2020-07-12-akka-streams-graphs.md
@@ -0,0 +1,172 @@
+---
+title: "How to Use Akka Streams Graph DSL"
+date: 2020-07-12
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka streams]
+excerpt: "Akka Streams has the Graph DSL, which is one of its most powerful APIs. Learn how to use it and get started quickly with Akka Streams."
+---
+
+This article is for the Scala programmer with a little bit of Akka background, but who is utterly baffled by the seemingly magical Akka Streams Graph DSL. This article will attempt to demystify the apparent magic of the DSL into bits of Scala code that are easy to understand.
+
+If you want to code with me, add the following to your build.sbt file:
+
+```scala
+val akkaVersion = "2.6.5"
+
+libraryDependencies ++= Seq(
+ // akka streams
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion,
+ "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion,
+)
+```
+
+## Prologue
+
+In this article I'm assuming you know what Akka tries to do with actors and a bit of Akka Streams, at least what a Source, Sink and Flow do. To recap, a basic usage example of Akka Streams is to create some (potentially async) components that deal with receiving and/or passing data around, and plugging them together like pipes:
+
+```scala
+implicit val system = ActorSystem()
+import system.dispatcher // "thread pool"
+
+// a source element that "emits" integers
+val source = Source(1 to 1000)
+// a flow that receives integers, transforms them and passes their results further down
+val flow = Flow[Int].map(x => x * 2)
+// a sink that receives integers and prints each to the console
+val sink = Sink.foreach[Int](println)
+// combine all components together in a static graph
+val graph = source.via(flow).to(sink)
+// start the graph = "materialize" it
+graph.run()
+```
+
+This example will print the number 2 to 2000, in steps of 2, on separate lines. Akka Streams is all about creating these individual data-receiving and data-passing components, and combining them to create useful data pipes in your application. The beauty of this way of thinking about data is that you don't need to concern yourself with synchronization or communication problems; everything is taken care of by the middleware.
+
+## Enter the Graph DSL
+
+After understanding the basic flow of Akka (pun intended), I see lots of people hitting a brick wall when they move to level 2 of Akka Streams, which is the infamous Graph DSL. It's not an API, it's a full-blown DSL.
+
+I called this Graph DSL a "level 2" of Akka Streams because besides the standard Source-flow-sink structure, complex applications need complex data-passing architectures. For example, when you're hitting reply-all to that email, you want to broadcast that email to all 354 recipients of your liss. Or another example: if you're writing an online store, and a user just paid for a product, you want to query the payment provider to make sure the payment went through, and at the same time instruct the fulfilment center to dispatch an order. These are likely two different services, whose replies you need to glue together before you can send your lovely user an order confirmation. These examples (and countless others) need something more than the linear source-flow-sink data passing scheme.
+
+Enter the Graph DSL.
+
+Let's do with something simple. Assume you are starting with a source of integers, and you want to feed them to two independent, asynchronous complex computation engines. Then you want to stitch the results together as a tuple, and print the tuples to the console. Almost inevitably, most documentation articles give you the whole structure which has some alien tokens inside, but I want to break this down into some steps that you can follow when you want to create your own graphs.
+
+First of all, you need to know what kind of components you will need for such a job. Besides sources, sinks and flows, we can have other components that can have either multiple inputs (like a zip) or multiple outputs (like a broadcast). Our graph will look something like below, with the components separated:
+
+
+
+This visual representation will be reflected in the code, much more than you think. So the above was step 0: make a mental diagram of how you want your data to move.
+
+## Step 1: the frame
+
+```scala
+val graph = GraphDSL.create() { implicit builder: GraphDSL.Builder[NotUsed] =>
+ import GraphDSL.Implicits._ // brings some nice operators in scope
+
+}
+```
+
+The `GraphDSL.create` is a curried function. The first argument list is empty (but it also has overloads with arguments, don't worry about those) and the second argument is a function. That function takes a mutable data structure called a Builder which is typed with a materialized value , which in our case will be NotUsed, as we aren't surfacing anything outside of the stream. Inside the function block, we are already importing `import GraphDSL.Implicits._` to bring some alien operators in scope.
+
+## Step 2: create the building blocks
+
+After the implicit import, still inside the block of the function, we need to add the individual components that we are going to use in the graph:
+
+```scala
+val input = builder.add(Source(1 to 1000)) // the initial elements
+val incrementer = builder.add(Flow[Int].map(x => x + 1)) // hard computation #1
+val multiplier = builder.add(Flow[Int].map(x => x * 10)) // hard computation #2
+val output = builder.add(Sink.foreach[(Int, Int)](println)) // the printer
+
+val broadcast = builder.add(Broadcast[Int](2)) // fan-out operator
+val zip = builder.add(Zip[Int, Int]) // fan-in operator
+```
+
+The last two components are the most interesting. The Broadcast has the capacity to duplicate the incoming data into multiple outputs - which will be fed into our individual super-heavy computations - and the Zip will receive the results from the two flows, and whenever a value is ready at both its inputs, it will take them, tuple them and send them downstream, while keeping the order of elements.
+
+## Step 3: glue the components together
+
+This step is the most fun and also the hardest to understand if you've never seen this before.
+
+```scala
+input ~> broadcast
+
+broadcast.out(0) ~> incrementer ~> zip.in0
+broadcast.out(1) ~> multiplier ~> zip.in1
+
+zip.out ~> output
+```
+
+The squiggly arrow thing is a method which is brought in scope by our implicits import in step 1 - we're of course using it infix because it looks cool. You might notice that we aren't using the result of these expressions. That's because the methods return Unit, but they take the implicit Builder (again from step 1) as argument. In other words, the squiggly arrow mutates the Builder which (internally) describes the layout of our stream. This step is one of the most powerful in Akka Streams, because the code looks visually similar to our earlier diagram. If I change some whitespace, I could make the code like this:
+
+```scala
+ broadcast.out(0) ~> incrementer ~> zip.in0
+input ~> broadcast; zip.out ~> output
+ broadcast.out(1) ~> multiplier ~> zip.in1
+```
+
+The code looks visual, all without needing to care about the internal implementation of these individual components. They're all asynchronous and backpressured, and we're getting all the benefits for free.
+
+## Step 4: closing
+
+Still in the block of the function we opened in step 1, we need to make the function return what Akka Streams calls a Shape. Because our graph is closed, i.e. has no open inputs and outputs, we're returning ClosedShape.
+
+```scala
+val graph = GraphDSL.create() { implicit builder: GraphDSL.Builder[NotUsed] =>
+ import GraphDSL.Implicits._ // brings some nice operators in scope
+
+ // ... the rest of the implementation
+
+ ClosedShape
+}
+```
+
+ClosedShape is an object which is a marker for Akka when you will materialize this graph, to make sure you didn't leave any internal component with any input or output hanging or unconnected.
+
+## Final code
+
+After you are done creating the graph, you will need to materialize it to run:
+
+```scala
+import akka.NotUsed
+import akka.actor.ActorSystem
+import akka.stream.ClosedShape
+import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, RunnableGraph, Sink, Source, Zip}
+
+object AkkaStreamsGraphs {
+
+ implicit val system = ActorSystem("GraphBasics")
+
+ // step 1 - setting up the fundamentals for the graph
+ val graph =
+ GraphDSL.create() { implicit builder: GraphDSL.Builder[NotUsed] => // builder = MUTABLE data structure
+ import GraphDSL.Implicits._ // brings some nice operators into scope
+
+ // step 2 - add the necessary components of this graph
+ val input = builder.add(Source(1 to 1000))
+ val incrementer = builder.add(Flow[Int].map(x => x + 1)) // hard computation
+ val multiplier = builder.add(Flow[Int].map(x => x * 10)) // hard computation
+ val output = builder.add(Sink.foreach[(Int, Int)](println))
+
+ val broadcast = builder.add(Broadcast[Int](2)) // fan-out operator
+ val zip = builder.add(Zip[Int, Int]) // fan-in operator
+
+ // step 3 - tying up the components
+ input ~> broadcast
+
+ broadcast.out(0) ~> incrementer ~> zip.in0
+ broadcast.out(1) ~> multiplier ~> zip.in1
+
+ zip.out ~> output
+
+ // step 4 - return a closed shape
+ ClosedShape
+ }
+
+ def main(args: Array[String]): Unit = {
+ RunnableGraph.fromGraph(graph).run()
+ }
+}
+```
diff --git a/_posts/2020-07-31-null-nothing-none-nil-unit.md b/_posts/2020-07-31-null-nothing-none-nil-unit.md
new file mode 100644
index 000000000000..4302c5e1d7d0
--- /dev/null
+++ b/_posts/2020-07-31-null-nothing-none-nil-unit.md
@@ -0,0 +1,121 @@
+---
+title: "The Difference Between Null, Nothing, Nil, None and Unit in Scala"
+date: 2020-07-31
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "We explore the various flavors of nothing-ness in Scala."
+---
+
+There are lots of terms to express the lack of values in Scala. Today we're going to discuss the difference between null, Null (not a typo), None, Nil, Unit and Nothing. They all seemingly describe the lack of a meaningful value, which can definitely be confusing. Thank goodness Scala doesn't have `undefined`...
+
+This article is for the beginner Scala programmer. We'll make a clear distinction between all these terms, and by the end of this article you'll know which to use when.
+
+
+## The null reference
+This is probably the most familiar. If you come to Scala from another programming language, perhaps very quickly , you're probably familiar with the `null` reference. It's used as an absent value.
+
+```scala
+val anAbsentString: String = null
+```
+
+You've surely dealt with your own set of null-access exceptions. That is because an absent value doesn't have fields or methods.
+
+```scala
+anAbsentString.length() // triggers the famous NullPointerException
+```
+
+This one is easy.
+
+## The Null type
+
+Now, in Scala, the null reference belongs to its own distinct type, which is called `Null` with a capital N.
+
+```scala
+val theNullReference: Null = null
+```
+
+The Null type has no methods, no fields, cannot be extended or instantiated, so by itself is pretty boring. The only interesting aspect of Null is that it "extends" all reference types. By that we mean that we can successfully use it as a replacement for any reference type
+
+```scala
+val noString: String = theNullReference
+val noPerson: Person = theNullReference
+val noList: List[Int] = theNullReference
+```
+
+So from the point of view of subtyping, Null is a proper subtype of all reference types. In Scala, the reference type hierarchy starts with AnyRef at the top and ends with Null at the bottom.
+
+A common question is: how can Null be a proper subtype for all reference types, since Scala offers a single-class inheritance model? The answer is that Null is treated in a particular way by the compiler, so there's no need for us programmers to intervene in any way.
+
+## Nil
+
+This sounds very similar to null and Null, but Nil means something completely different: Nil is the empty implementation of a List.
+
+```scala
+val anEmptyList: List[Int] = Nil
+```
+
+Unlike null, Nil is a proper value. It has fields and methods:
+
+```scala
+val emptyListLength: Int = Nil.length
+```
+
+We can pass it around:
+
+```scala
+def processList(list: List[Int]): Int = list.length
+
+// later:
+procesesList(Nil)
+```
+
+and generally do with Nil whatever we do with regular values.
+
+## None
+
+Truth be told, we very rarely use `null` in Scala. Using null incentivizes us to write imperative, Java-style, defensive and error-prone code. Instead of `null`, we commonly use Options, which are data structures meant to represent the presence or absence of a value. Options allow (and force) us to write clearer, more concise code which is harder to fail. The two kinds of instances we can use for Option are `Some` instances and the `None` value.
+
+```scala
+val anAbsentInt: Option[Int] = None
+val aPresentInt: Option[Int] = Some(42)
+```
+
+The difference between `None` and `null` is that `None` is a proper value (much like `Nil`) and we can pass it around and process it. We're going to talk more about Options and why they are useful in another article.
+
+The `null` and `None` values are interoperable, in the sense that we can lift ourselves from the muddy null-checking realm to Options by using the Option apply factory method:
+
+```scala
+val anAbsentValue: Option[Int] = Option(null) // this returns None
+```
+
+## Unit
+
+Thankfully, Unit will hopefully prove a little clearer. One of the very first things we learn as Scala programmers coming from another language is how to declare methods returning "void". The equivalent of "void" in other languages is `Unit` in Scala.
+
+```scala
+def aUnitReturningFunction(): Unit = println("Starting to get the difference!")
+```
+
+If you are clear on the difference between `null` and `void` in other languages, you'll understand how `Unit` is different from what we talked so far.
+
+## Nothing
+
+Finally, let's end with the mother of nothingness. We've spoken about Nothing before so we won't spend too much time here, but Nothing is the type of no value at all. Nothing can't be instantiated and has no values of that type. Nothing truly means nothing: not even null, not Unit, not None, nothing at all. The only expressions that return Nothing are throwing exceptions:
+
+```scala
+val theNothing = throw new RuntimeException("Nothing to see here")
+```
+
+The interesting thing about Nothing is that, much like Null, it can be used as a replacement for any type - this time including the value types:
+
+```scala
+val nothingInt: Int = throw new RuntimeException("No int")
+val nothingString: String = throw new RuntimeException("No string")
+```
+
+In other words, Nothing is a proper subtype for all possible types in Scala. Much like Null, it's treated in a special way by the compiler.
+
+## Conclusion
+
+I hope that by now you have more clarity about the differences between null, Null, None, Nil, Unit and Nothing. Each is used for a completely different purpose, unlike some other languages' multiple null versions (cough* JavaScript cough*).
diff --git a/_posts/2020-08-04-twenty-seconds.md b/_posts/2020-08-04-twenty-seconds.md
new file mode 100644
index 000000000000..d66f70b63728
--- /dev/null
+++ b/_posts/2020-08-04-twenty-seconds.md
@@ -0,0 +1,111 @@
+---
+title: "How Does \"20 seconds\" Work in Scala?"
+date: 2020-08-04
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, tricks]
+excerpt: "Various constructions like \"20.seconds\" look like baked into the Scala language. Let's see how these work."
+---
+This short article will show you how apparently magical constructs like `20.seconds` works in Scala, although the Int type doesn't have such methods natively.
+
+This article will make more sense if you know the basics of implicits, but then again, if you do know how implicits work, there's only one step to understanding how these seemingly magical methods work, so I'll cover everything you need to know.
+
+## 1. The "Problem"
+
+The question we're addressing here is the following: the Int type has a very small set of methods and certainly the `seconds` method isn't one of them:
+
+```scala
+val womp = 20.seconds // compile error: symbol "seconds" not found
+```
+
+However, once we add a special import, it magically works:
+
+```scala
+import scala.concurrent.duration._
+
+val aDuration = 20.seconds // works!
+```
+
+So how does the magical import work?
+
+## 2. Enriching Types
+
+The answer is not in the import itself, but in what's imported - the types and values that are imported might as well be in scope and methods like `.seconds` would work just as fine. It's their structure that provides the magic. To understand how they work, we need to go back to implicits.
+
+I'm not going to talk about all the functionality that the `implicit` keyword does in Scala - we'll probably do that in another article - but we are going to focus on one kind of implicits: _implicit classes_. Implicit classes are one-argument wrappers, i.e. a class with one constructor argument, with regular methods, fields, etc, except that they have the `implicit` keyword in their declaration:
+
+```scala
+implicit class MyRichString(string: String) {
+ def fullStop: String = string + "."
+}
+```
+
+If I removed the `implicit` keyword there, this would be a pretty uninteresting class. Adding `implicit` will add some special powers. We can either say
+
+```scala
+new MyRichString("This is a sentence").fullStop
+```
+
+or, watch this:
+
+```scala
+"This is a sentence".fullStop
+```
+
+This works although the `fullStop` method doesn't exist for the String class. Normally, the code would not compile, but the compiler will add an extra step of searching for any implicit wrapping or conversion of a String value that might have the `fullStop` method, which in our case it does. So in reality, the compiler will rewrite our last call as
+
+```scala
+// "This is a sentence".fullStop beecomes:
+new MyRichString("This is a sentence").fullStop
+```
+
+which is what we (explicitly) wrote earlier. This pattern provides what we call extension methods - libraries like Cats use this all the time.
+
+## 3. Importing
+
+If an implicit class like this is not written in the scope where we use the "magical" method, the code will not compile until we bring that implicit class into scope. This means an import. Usually, libraries (including the standard library) packs implicits into "Ops"-like objects:
+
+```scala
+package mylibrary
+
+object MyStringOps {
+ implicit class MyRichString(string: String) {
+ def fullStop: String = string + "."
+ }
+}
+
+```
+
+and then later in our code, when we import it, we'll also have access to the extension method:
+
+```scala
+import mylibrary._
+
+"Starting to get it".fullStop
+```
+
+The Scala `duration` package works in the same way: when you import `scala.concurrent.duration._` you gain access to extension methods on the Int type that returns instances of `Duration`:
+
+```scala
+import scala.concurrent.duration._
+20.seconds
+500.millis
+```
+
+## 4. A Bit More and Conclusion
+
+Some packages are automatically imported with every Scala code. Some of these packages might include extension methods like the `.seconds` one. Ever wondered how things like
+
+```scala
+val range = 1 to 100
+```
+
+work? With the tools you now have, it's quite easy to understand that there is an implicit conversion that enriches the Int type:
+
+```scala
+// some implicit conversion here { ...
+ def to(end: Int): Range
+// ... }
+```
+
+You've now learned how extension methods work and how magical things like `20.seconds` work in Scala, making them seem like they're part of the language or standard library. This skill will prove useful as you become more experienced with Scala libraries like Cats.
diff --git a/_posts/2020-08-07-abstract-classes-vs-traits.md b/_posts/2020-08-07-abstract-classes-vs-traits.md
new file mode 100644
index 000000000000..021373a3d0b0
--- /dev/null
+++ b/_posts/2020-08-07-abstract-classes-vs-traits.md
@@ -0,0 +1,74 @@
+---
+title: "The Practical Difference Between Abstract Classes and Traits in Scala"
+date: 2020-08-07
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "Abstract classes and traits share a lot of overlap in Scala. How are they actually different?"
+---
+This short article will compare abstract classes and traits as means of inheritance. This is for the beginner Scala programmer who is just getting started with Scala's inheritance model. If you're trying to figure out which is the best way to create OO-style type hierarchies and abstract classes seem too similar to traits, this article is for you.
+
+## Background
+
+Scala is particularly permissive when it comes to inheritance options. Of course, Scala has a single-class inheritance model, where one can extend a single class. Much like Java, Scala classes can inherit from multiple traits (the equivalent of Java interfaces).
+
+Scala also has the concept of an abstract class, where one can
+ - restrict the ability of a class to be instantiated
+ - add unimplemented fields or methods
+
+```scala
+abstract class Person {
+ // the abstract keyword is not necessary in the field/method definition
+ val canDrive: Boolean
+ def discussWith(another: Person): String
+}
+
+val bob = new Person // error - not all members implemented
+```
+
+An abstract class may or may not have either abstract (unimplemented) or non-abstract (implemented) methods.
+
+However, traits also share most of the same capabilities:
+ - they can't be instantiated by themselves
+ - they may have abstract (unimplemented) fields or methods
+ - (more importantly) they may have non-abstract (implemented) fields and methods as well
+
+So while teaching Scala at Rock the JVM, I often come across this popular question: how are traits and abstract classes really different?
+
+## The similarity
+
+There is little practical difference between an abstract class and a trait if you extend a single one. If you look at the example below:
+
+```scala
+class Adult(val name: String, hasDrivingLicence: Boolean) extends Person {
+ override def toString: String = name
+ override val canDrive: Boolean = hasDrivingLicence
+ override def discussWith(another: Person): String = "Indeed, ${other}, Kant was indeed revolutionary for his time..."
+}
+```
+
+the code is identical, regardless of whether Person was a trait or an abstract class. So from this point of view, a trait and an abstract class are identical.
+
+## The practical differences
+
+When we move towards multiple inheritance, things start to look different. One of the fundamental differences between a trait and an abstract class is that multiple traits may be used in the inheritance definition of a class, whereas a single abstract class can be extended. Since traits can have non-abstract methods as well, a good (and natural) follow-up question might be: "but what happens if a class inherits from two traits with the same method implemented in different ways?", which resembles the famous diamond problem. Scala solves it with a dedicated mechanism called linearization, which we will discuss in another article.
+
+A second, more practical difference between a trait and an abstract class is that an abstract class can take constructor arguments:
+
+```scala
+abstract class Person(name: String, age: Int)
+```
+
+whereas traits cannot:
+
+```scala
+trait Person(name: String, age: Int) // error
+```
+
+That is, at least until Scala 3 comes out - Scala 3 will also allow constructor arguments for traits.
+
+## The subtle difference
+
+When I speak in a live training about the above differences between traits and abstract classes, my audience is usually not impressed - it seems the overlap is so large that there might be no purpose to two different concepts.
+
+However, I always end up with the following reasoning: for good practice, it is always a good idea to make code easy to read, understand and reason about. To this aim, we generally represent "things" as classes, and "behaviors" as traits. This conceptual difference is directly mapped in the language: a "thing" (class) can be a specialized version of another "thing" (also a class), but might incorporate multiple "behaviors" (traits).
diff --git a/_posts/2020-08-10-type-level-programming-part-1.md b/_posts/2020-08-10-type-level-programming-part-1.md
new file mode 100644
index 000000000000..d7dec343bf56
--- /dev/null
+++ b/_posts/2020-08-10-type-level-programming-part-1.md
@@ -0,0 +1,167 @@
+---
+title: "Type-Level Programming in Scala, Part 1"
+date: 2020-08-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "This one is hardcode. We make the Scala compiler infer type relationships for us at compile time. Pure power."
+---
+
+In this mini-series I'll introduce you to some advanced Scala techniques for type-level programming. In other words, you'll learn to use the power of the Scala compiler to solve problems for you and to validate properties of types .
+
+This article (and its subsequent parts) require an advanced Scala programmer, so you'll need to be comfortable with quite a few of the harder topics in Scala:
+ - type aliases
+ - type members
+ - implicit methods
+ - how the compiler searches for implicits
+
+In this first part of the series, I'll introduce you the notion of values as types and I'll show how you can embed a mathematical problem as a type constraint, which the Scala compiler can happily validate or invalidate for you before the code can compile.
+
+## Prologue
+
+This article series is applicable to Scala 2.12 and 2.13 - when Scala 3 arrives, I'll release an updated Scala 3 version of this series. If you're following this article series on Scala 2.13, make sure you add the scala-reflect package to your build.sbt file:
+
+```scala
+libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value
+```
+
+This is not a cheat - we aren't using reflection to manipulate types at runtime. We're using the reflect package to print something to the console, although the mere fact the code compiles will mean that we're successful in what we're going to do. Nevertheless, I'll add the following starter code here so we can focus only on type-level programming from now on:
+
+```scala
+package myPackage
+
+object TypeLevelProgramming { // this is where I'll store my main method
+ import scala.reflect.runtime.universe._
+ def show[T](value: T)(implicit tag: TypeTag[T]) =
+ tag.toString.replace("myPackage.TypeLevelProgramming.", "") // this will be very verbose otherwise
+}
+```
+
+With that, we're ready to dive into type-level programming with Scala.
+
+## 1. Numbers as Types
+
+We're going to start easy. We're going to declare a type that will represent natural numbers. However, we aren't going to use the plain Int type, but rather we'll represent the relationship between numbers as a succession.
+
+```scala
+trait Nat
+class _0 extends Nat
+class Succ[A <: Nat] extends Nat
+```
+
+With the above code, we've defined the entire set of natural numbers! Think about it. If you wanted to represent the number 5, how would you do it in terms of the above definitions, in Scala?
+
+```scala
+type _1 = Succ[_0]
+type _2 = Succ[_1] // = Succ[Succ[_0]]
+type _3 = Succ[_2] // = Succ[Succ[Succ[_0]]]
+type _4 = Succ[_3]
+type _5 = Succ[_4]
+```
+
+Essentially, the number 5 is now a type , and we represent it as `Succ[Succ[Succ[Succ[Succ[_0]]]]]]`. The compiler can represent any number at all in terms of this succession relationship. We'll keep the above type aliases for convenience.
+
+This natural number representation bears the name of the would-be-rockstar-Scala-functional-programmer-if-he-were-alive Giuseppe Peano , and the number relationships that we're going to transform into type constraints in the Scala compiler will be the foundation of the Peano number arithmetic.
+
+## 2. Number Comparison as Types
+
+The first part was gentle enough. If I've caught your attention and curiosity, then do stick around - we're in for quite a ride.
+
+At this stage, we have numbers and their succession relationship as type constraints. The number 1 is the successor of 0, 2 is the successor of 1, and so on. We'd like to be able to tell whether one number is "less than" another. This relationship is not trivial. Succession (difference of 1) is fine, but how can you tell that 1 is "less than" 4 just by looking at the type definition?
+
+We'll make the compiler determine that for us.
+
+```scala
+trait <[A <: Nat, B <: Nat]
+```
+
+You're probably well aware that Scala is very permissive with naming methods, identifiers and types, and `<` is perfectly fine (and convenient). We can, of course, use this as a regular type:
+
+```scala
+val someComparison: _2 < _3 = ???
+```
+
+But we won't need that, because the compiler will create the appropriate instances for us.
+
+## 3. The Compiler Validates
+
+We'll never build instances of "less-than" ourselves, but we'll make the compiler build implicit instances of "less-than" just for the right types . Through these instances, we'll thus validate the existence of the "less-than" type for various numbers, and therefore prove the comparison between numbers.
+
+Implicit methods to the rescue:
+
+```scala
+implicit def ltBasic[B <: Nat]: <[_0, Succ[B]] = new <[_0, Succ[B]] {}
+```
+
+What does this mean? For every type B which extends Nat, the compiler will be able to automatically build an instance of `_0 < Succ[B]`. This will have the same meaning as Peano's first axiom of comparison: for every natural n, 0 < succ(n).
+
+We can write a similar thing to make the compiler automatically compare other numbers:
+
+```scala
+implicit def inductive[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[Succ[A], Succ[B]] = new <[Succ[A], Succ[B]] {}
+```
+
+In other words: if the compiler can find and implicit instance of `<[A, B]`, then the compiler will also build an instance of `<[Succ[A], Succ[B]]`. This mechanism has the same effect as proving the second of Peano's axioms on comparison: for every two naturals a and b, if a < b, then succ(a) < succ(b).
+
+These two implicit methods, used in combination, will have the powerful effect of proving the existence of any `<[A,B]` type, whenever the "less-than" relationship makes sense. Let's add a simple method which fetches the implicit instance of comparison at the point of use, and group the implicit methods into the companion of `<` because the compiler will search that space in its attempt to resolve implicits:
+
+```scala
+object < {
+ def apply[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[A, B] = lt
+ implicit def ltBasic[B <: Nat]: <[_0, Succ[B]] = new <[_0, Succ[B]] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[Succ[A], Succ[B]] = new <[Succ[A], Succ[B]] {}
+}
+```
+
+In this way, we can write the following:
+
+```scala
+val validComparison: _2 < _3 = <[_2, _3]
+```
+
+This compiles, because
+ - the apply method needs an implicit instance of `<[_2, _3]`
+ - to find that instance, the compiler can choose to run any of the two implicit methods - it will attempt to call the `inductive` method, but it will need an implicit instance of `<[_1, _2]`
+ - in the same way, the compiler marks that it can call the `inductive` method, but it needs an implicit instance of type `<[_0, _1]`
+ - in this case, the method signature of `ltBasic` signals that the compiler can build an instance of `<[_0, _1]` because `_1 = Succ[0]`
+ - now, given an instance of `<[_0, _1]` the compiler can build an instance of `<[_1, _2]`
+ - in the same style, given an instance of `<[_1, _2]` the compiler can build an instance of `<[_2, _3]`
+ - given the instance of `<[_2, _3]`, it can safely be passed to the `apply` method and returned
+
+However, if you try to wrote
+
+```scala
+val invalidComparison: _3 < _2 = <[_3, _2]
+```
+
+this will trigger a compiler error, because no implicit instances of `<[_3, _2]` can be found.
+
+## 4. Number Comparison as Types
+
+The full code, with a shameless copy of less-than-equal, is below:
+
+```scala
+ trait Nat
+ class _0 extends Nat
+ class Succ[A <: Nat] extends Nat
+
+ type _1 = Succ[_0]
+ type _2 = Succ[_1] // = Succ[Succ[_0]]
+ type _3 = Succ[_2] // = Succ[Succ[Succ[_0]]]
+ type _4 = Succ[_3] // ... and so on
+ type _5 = Succ[_4]
+
+ sealed trait <[A <: Nat, B <: Nat]
+ object < {
+ def apply[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[A, B] = lt
+ implicit def ltBasic[B <: Nat]: <[_0, Succ[B]] = new <[_0, Succ[B]] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[Succ[A], Succ[B]] = new <[Succ[A], Succ[B]] {}
+ }
+
+ sealed trait <=[A <: Nat, B <: Nat]
+ object <= {
+ def apply[A <: Nat, B <: Nat](implicit lte: <=[A, B]): <=[A, B] = lte
+ implicit def lteBasic[B <: Nat]: <=[_0, B] = new <=[_0, B] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lt: <=[A, B]): <=[Succ[A], Succ[B]] = new <=[Succ[A], Succ[B]] {}
+ }
+```
diff --git a/_posts/2020-08-17-type-level-programming-part-2.md b/_posts/2020-08-17-type-level-programming-part-2.md
new file mode 100644
index 000000000000..204e7b3a7de8
--- /dev/null
+++ b/_posts/2020-08-17-type-level-programming-part-2.md
@@ -0,0 +1,292 @@
+---
+title: "Type-Level Programming in Scala, Part 2"
+date: 2020-08-17
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "We continue down the path of type-level power in Scala. We learn to add numbers as types, at compile time."
+---
+In this article we will continue what we started in the [first part](/type-level-programming-part-1) and enhance our type-level capabilities. As a quick reminder, in this mini-series we learn to use the power of the Scala compiler to validate complex relationships between types that mean something special to us, for example that a "number" (as a type) is "smaller than" another "number".
+
+## Background
+
+I mentioned earlier that I'm using Scala 2 - I'll update this series once Scala 3 arrives - so you'll need the following to your build.sbt:
+
+```scala
+libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value
+```
+
+and in order to print a type signature, I have the following code:
+
+```scala
+package myPackage
+
+object TypeLevelProgramming {
+ import scala.reflect.runtime.universe._
+ def show[T](value: T)(implicit tag: TypeTag[T]) =
+ tag.toString.replace("myPackage.TypeLevelProgramming.", "")
+}
+```
+
+Again, we aren't cheating so that we manipulate types at runtime - the compiler will figure out the types before the code is compiled, and we use the above code just to print types.
+
+## Where We Left Off
+
+The first article discussed the creation of natural numbers as types, following the Peano representation: zero as a starting value, then the succession of numbers as types.
+
+```scala
+trait Nat
+class _0 extends Nat
+class Succ[A <: Nat] extends Nat
+
+type _1 = Succ[_0]
+type _2 = Succ[_1] // = Succ[Succ[_0]]
+type _3 = Succ[_2] // = Succ[Succ[Succ[_0]]]
+type _4 = Succ[_3] // ... and so on
+type _5 = Succ[_4]
+```
+
+Then we built up a comparison relationship between these numbers, again as a type. This time, we also made the compiler build implicit instances of the "less-than" type in order to prove that the relationship is true:
+
+```scala
+sealed trait <[A <: Nat, B <: Nat]
+object < {
+ def apply[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[A, B] = lt
+ implicit def ltBasic[B <: Nat]: <[_0, Succ[B]] = new <[_0, Succ[B]] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[Succ[A], Succ[B]] = new <[Succ[A], Succ[B]] {}
+}
+
+sealed trait <=[A <: Nat, B <: Nat]
+object <= {
+ def apply[A <: Nat, B <: Nat](implicit lte: <=[A, B]): <=[A, B] = lte
+ implicit def lteBasic[B <: Nat]: <=[_0, B] = new <=[_0, B] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lt: <=[A, B]): <=[Succ[A], Succ[B]] = new <=[Succ[A], Succ[B]] {}
+}
+```
+
+In this way, if we write
+
+```scala
+val validComparison: <[_2, _3] = <[_2, _3]
+```
+
+the compiler can build an implicit instance of `<[_2, _3]` so the relationship is "correct" and the code can compile, whereas if we write
+
+```scala
+val invalidComparison: <[_3, _2] = <[_3, _2]
+```
+
+the code will not compile because the compiler will be unable to build an implicit instance of `<[_3, _2]`.
+
+We will use similar techniques to take the power up a notch: the ability to add and subtract "numbers" as types.
+
+## Adding "Numbers" as Types
+
+In a similar style that we used to compare numbers, we'll create another type that will represent the sum of two numbers as types:
+
+```scala
+trait +[A <: Nat, B <: Nat, S <: Nat]
+```
+
+which means that "number" A added with "number" B will give "number" S. First, we'll make the compiler automatically detect the truth value of A + B = S by making it construct an implicit instance of `+[A, B, S]`. We should be able to write
+
+```scala
+val four: +[_1, _3, _4] = +[_1, _3, _4]
+```
+
+but the code should not compile if we wrote
+
+```scala
+val five: +[_3, _1, _5] = +[_3, _1, _5]
+```
+
+so for starters, we should create a companion for the `+` trait and add an apply method to fetch whatever implicit instance can be built by the compiler:
+
+```scala
+object + {
+ def apply[A <: Nat, B <: Nat, S <: Nat](implicit plus: +[A, B, S]): +[A, B, S] = plus
+}
+```
+
+In the same companion object we will also build the implicit values that the compiler can work with.
+
+## Axioms of Addition as Implicits
+
+First, we know that 0 + 0 = 0. In terms of the trait that we defined, that means the compiler must have access to an instance of `+[_0, _0, _0]`. So let's build it ourselves:
+
+```scala
+implicit val zero: +[_0, _0, _0] = new +[_0, _0, _0] {}
+```
+
+That one was the easiest. Another "base" axiom is that for any number A > 0, it's always true that A + 0 = A and 0 + A = A. We can embed these axioms as implicit methods:
+
+```scala
+implicit def basicRight[A <: Nat](implicit lt: _0 < A): +[_0, A, A] = new +[_0, A, A] {}
+implicit def basicLeft[A <: Nat](implicit lt: _0 < A): +[A, _0, A] = new +[A, _0, A] {}
+```
+
+Notice that the type `+[_0, A, A]` and `+[A, _0, A]` are different - this is why we need two different methods there.
+
+At this point, you might be wondering why we need those constraints and why we can't simply say that for any number A, it's always true that 0 + A = A and A + 0 = A. That's true and more general, but it's also confusing to the compiler, because it would have multiple routes through which it can build an instance of +[_0, _0, _0]. We want to separate these cases so that the compiler can build each implicit instance by following exactly one induction path.
+
+With these 3 implicits, we can already validate a number of sums:
+
+```scala
+val zeroSum: +[_0, _0, _0] = +[_0, _0, _0]
+val anotherSum: +[_0, _2, _2] = +[_0, _2, _2]
+```
+
+but are yet to validate sums like `+[_2, _3, _5]`. That's the subject of the following inductive axiom. The reasoning works like this: if A + B = S, then it's also true that Succ[A] + Succ[B] = Succ[Succ[S]]. In the compiler's language, if the compiler can create an implicit +[A, B, S], then it must als be able to create an implicit +[Succ[A], Succ[B], Succ[Succ[S]]]:
+
+```scala
+implicit def inductive[A <: Nat, B <: Nat, S <: Nat](implicit plus: +[A, B, S]): +[Succ[A], Succ[B], Succ[Succ[S]]] =
+ new +[Succ[A], Succ[B], Succ[Succ[S]]] {}
+```
+
+With these 4 implicits, the compiler is now able to validate any sum. For example:
+
+```scala
+val five: +[_2, _3, _5] = +[_2, _3, _5]
+```
+
+This compiles, because
+ - the compiler needs an implicit `+[_2, _3, _5]` which is in fact `+[Succ[_1], Succ[2], Succ[Succ[_3]]]`
+ - the compiler can run the inductive method, but it requires an implicit `+[_1, _2, _3]` which is `+[Succ[_0], Succ[_1], Succ[Succ[_1]]]`
+ - the compiler can run the inductive method again, but it requires an implicit `+[_0, _1, _1]`
+ - the compiler can run the basicRight method and build the implicit `+[_0, _1, _1]`
+ - the compiler can then build all the other dependent implicits
+
+However, if we write an incorrect "statement", such as
+
+```scala
+val four: +[_2, _3, _4] = +[_2, _3, _4]
+```
+
+the code can't compile because the compiler can't find the appropriate implicit instance.
+
+Right now, our code looks like this:
+
+```scala
+trait +[A <: Nat, B <: Nat, S <: Nat]
+object + {
+ implicit val zero: +[_0, _0, _0] = new +[_0, _0, _0] {}
+ implicit def basicRight[B <: Nat](implicit lt: _0 < B): +[_0, B, B] = new +[_0, B, B] {}
+ implicit def basicLeft[B <: Nat](implicit lt: _0 < B): +[B, _0, B] = new +[B, _0, B] {}
+ implicit def inductive[A <: Nat, B <: Nat, S <: Nat](implicit plus: +[A, B, S]): +[Succ[A], Succ[B], Succ[Succ[S]]] =
+ new +[Succ[A], Succ[B], Succ[Succ[S]]] {}
+ def apply[A <: Nat, B <: Nat, S <: Nat](implicit plus: +[A, B, S]): +[A, B, S] = plus
+}
+```
+
+## Supercharging Addition
+
+This is great so far! We can make the compiler validate type relationships like the "addition" of "numbers" at compile time. However, at this point we can't make the compiler figure out what the result of an addition should be - we need to specify the result type ourselves, and the compiler will simply show a thumbs-up if the type is good.
+
+The next level in this Peano arithmetic implementation would be to somehow make the compiler infer the sum type by itself. For that, we'll change the type signature of the addition:
+
+```scala
+trait +[A <: Nat, B <: Nat] extends Nat {
+ type Result <: Nat
+}
+```
+
+So instead of a type argument, we now have an abstract type member. This will help us when we use the `+` type at the testing phase. Now, in the companion object, we'll declare an auxiliary type:
+
+```scala
+object + {
+ type Plus[A <: Nat, B <: Nat, S <: Nat] = +[A, B] { type Result = S }
+}
+```
+
+This new type `Plus` is exactly the same as our previous `+` and we will use it in our implicit resolution. The trick here is to have the compiler automatically match the `Result` abstract type member to the `S` type argument of the auxiliary sum type.
+
+The next step is to change our axiom (read: implicits) definitions to use this new type:
+
+```scala
+implicit val zero: Plus[_0, _0, _0] = new +[_0, _0] { type Result = _0 }
+implicit def basicRight[B <: Nat](implicit lt: _0 < B): Plus[_0, B, B] = new +[_0, B] { type Result = B }
+implicit def basicLeft[B <: Nat](implicit lt: _0 < B): Plus[B, _0, B] = new +[B, _0] { type Result = B }
+implicit def inductive[A <: Nat, B <: Nat, S <: Nat](implicit plus: Plus[A, B, S]): Plus[Succ[A], Succ[B], Succ[Succ[S]]] =
+ new +[Succ[A], Succ[B]] { type Result = Succ[Succ[S]] }
+```
+
+Each rewrite goes as follows:
+ - make the return type be the new `Plus` type instead of the old `+`
+ - because we can't build `Plus` directly, we'll need to build an instance of `+` that has the correct type member
+
+Finally, the apply method will need to undergo a change as well. First of all, we'll get rid of the third type argument:
+
+```scala
+def apply[A <: Nat, B <: Nat](implicit plus: +[A, B]): +[A, B] = plus
+```
+
+At this point, we can now say
+
+```scala
+val five: +[_2, _3] = +.apply // or +[_2, _3]
+```
+
+and if the code compiles, then the compiler is able to validate the existence of a sum type between _2 and _3.
+
+But what's the result?
+
+## The Final Blow
+
+Right now, we can't see the final result of summing the "numbers". If we print the type tag of the sum we won't get too much info:
+
+```scala
+> println(show(+[_2, _3]))
+TypeTag[_2 + _3]
+```
+
+However, we can force the compiler to show the result type to us, because we have a `Result` type member in the `+` trait. All we need to do is change the apply method slightly:
+
+```scala
+def apply[A <: Nat, B <: Nat](implicit plus: +[A, B]): Plus[A, B, plus.Result] = plus
+```
+
+Instead of returning a `+[A, B]`, we return a `Plus[A, B, plus.Result]`. We can use this dirty trick because
+ - `Plus` is nothing but a type alias
+ - we can use type members in method return types
+
+With this minor change, the code still compiles, but if we show the type tag now, the tag looks different:
+
+```scala
+> println(show(+[_2, _3]))
+TypeTag[Succ[Succ[_0]] + Succ[Succ[Succ[_0]]]{ type Result = Succ[Succ[Succ[Succ[Succ[_0]]]]] }]
+```
+
+In other words, the result type the compiler has is `Succ[Succ[Succ[Succ[Succ[_0]]]]]`, which is `_5`!
+
+The final code is below:
+
+```scala
+trait +[A <: Nat, B <: Nat] { type Result <: Nat }
+object + {
+ type Plus[A <: Nat, B <: Nat, S <: Nat] = +[A, B] { type Result = S }
+ implicit val zero: Plus[_0, _0, _0] = new +[_0, _0] { type Result = _0 }
+ implicit def basicRight[B <: Nat](implicit lt: _0 < B): Plus[_0, B, B] = new +[_0, B] { type Result = B }
+ implicit def basicLeft[B <: Nat](implicit lt: _0 < B): Plus[B, _0, B] = new +[B, _0] { type Result = B }
+ implicit def inductive[A <: Nat, B <: Nat, S <: Nat](implicit plus: Plus[A, B, S]): Plus[Succ[A], Succ[B], Succ[Succ[S]]] =
+ new +[Succ[A], Succ[B]] { type Result = Succ[Succ[S]] }
+ def apply[A <: Nat, B <: Nat](implicit plus: +[A, B]): Plus[A, B, plus.Result] = plus
+}
+
+def main(args: Array[String]): Unit = {
+ println(show(+[_2, _3]))
+}
+```
+
+P.S. Creating a value of the sum type and then printing it will not produce the same result because of how types are attached to expressions:
+
+```scala
+> val five: +[_2, _3] = +[_2, _3]
+> println(five)
+TypeTag[_2 + _3]
+```
+
+## To Be Continued
+
+I hope that by the end of this second part you're getting a sense of how you can express a mathematical problem as a set of constraints on type relationships, written as implicit definitions. The compiler "solves" the problem for you by automatically building the right implicit, whose type holds the relationship you wanted.
+
+There's more to come...
diff --git a/_posts/2020-08-25-type-level-programming-part-3.md b/_posts/2020-08-25-type-level-programming-part-3.md
new file mode 100644
index 000000000000..35ee4e47f7d8
--- /dev/null
+++ b/_posts/2020-08-25-type-level-programming-part-3.md
@@ -0,0 +1,450 @@
+---
+title: "Type-Level Programming in Scala, Part 3"
+date: 2020-08-25
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "Final blow in the type-level trilogy. We learn how to sort lists... at compile time."
+---
+In this article we will continue what we started in the [first](/type-level-programming-1) and [second](/type-level-programming-2) parts of the series and become a badass type-level programmer in Scala. This mini-series is about using the power of the Scala compiler to enforce complex relationships between types that mean something special to us. In the first part , we defined the "less-than" relationship between numbers as types, and in the second part we learned to "add" two "numbers" as types. Everything, again, at compile time.
+
+The third time is the charm. In this third part, we're going to do something magical. We're going to rely on the first part (with number-type comparisons) and sort lists... at compile time.
+
+## Background
+
+I mentioned earlier that I'm using Scala 2 - I'll update this series once Scala 3 arrives - so you'll need the following to your build.sbt:
+
+```scala
+libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value
+```
+
+and in order to print a type signature, I have the following code:
+
+```scala
+package myPackage
+
+object TypeLevelProgramming {
+ import scala.reflect.runtime.universe._
+ def show[T](value: T)(implicit tag: TypeTag[T]) =
+ tag.toString.replace("myPackage.TypeLevelProgramming.", "")
+}
+```
+
+Again, we aren't cheating so that we manipulate types at runtime - the compiler will figure out the types before the code is compiled, and we use the above code just to print types.
+
+## Where We Left Off
+
+We are considering numbers as types, in the Peano representation: zero as the starting number, and every number defined in terms of its succession to another number.
+
+```scala
+trait Nat
+class _0 extends Nat
+class Succ[A <: Nat] extends Nat
+
+type _1 = Succ[_0]
+type _2 = Succ[_1] // = Succ[Succ[_0]]
+type _3 = Succ[_2] // = Succ[Succ[Succ[_0]]]
+type _4 = Succ[_3] // ... and so on
+type _5 = Succ[_4]
+```
+
+We then built a comparison relationship as a type, while making the compiler build implicit instances of that type so that it can "prove" than one "number" is "less than" another. The code looked like this:
+
+```scala
+ trait <[A <: Nat, B <: Nat]
+ object < {
+ implicit def ltBasic[B <: Nat]: <[_0, Succ[B]] = new <[_0, Succ[B]] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lt: <[A, B]): <[Succ[A], Succ[B]] = new <[Succ[A], Succ[B]] {}
+ def apply[A <: Nat, B <: Nat](implicit lt: <[A, B]) = lt
+ }
+ val comparison: _1 < _3 = <[_1, _3]
+ /*
+ What the compiler does:
+
+ <.apply[_1, _3] -> requires implicit <[_1, _3]
+ inductive[_0, _2] -> requires implicit <[_0, _2]
+ ltBasic[_1] -> produces implicit <[_0, Succ[_1]] == <[_0, _2]
+ */
+ // val invalidComparison: _3 < _2 = <[_3, _2] - will not compile: 3 is NOT less than 2
+
+ trait <=[A <: Nat, B <: Nat]
+ object <= {
+ implicit def lteBasic[B <: Nat]: <=[_0, B] = new <=[_0, B] {}
+ implicit def inductive[A <: Nat, B <: Nat](implicit lte: <=[A, B]): <=[Succ[A], Succ[B]] = new <=[Succ[A], Succ[B]] {}
+ def apply[A <: Nat, B <: Nat](implicit lte: <=[A, B]) = lte
+ }
+ val lteTest: _1 <= _1 = <=[_1, _1]
+ // val invalidLte: _3 <= _1 = <=[_3, _1] - will not compile
+```
+
+The second part was about real arithmetic between numbers. We also embedded the addition between two number-types as a type:
+
+```scala
+ // this time, we'll embed the result as a type member
+ trait +[A <: Nat, B <: Nat] { type Result <: Nat }
+ object + {
+ type Plus[A <: Nat, B <: Nat, S <: Nat] = +[A, B] { type Result = S }
+ // 0 + 0 = 0
+ implicit val zero: Plus[_0, _0, _0] = new +[_0, _0] { type Result = _0 }
+ // for every A <: Nat st A > 0, we have A + 0 = A and 0 + A = A
+ implicit def basicRight[A <: Nat](implicit lt: _0 < A): Plus[_0, A, A] = new +[_0, A] { type Result = A }
+ implicit def basicLeft[A <: Nat](implicit lt: _0 < A): Plus[A, _0, A] = new +[A, _0] { type Result = A }
+ // if A + B = S, then Succ[A] + Succ[B] = Succ[Succ[S]]
+ implicit def inductive[A <: Nat, B <: Nat, S <: Nat](implicit plus: Plus[A, B, S]): Plus[Succ[A], Succ[B], Succ[Succ[S]]] =
+ new +[Succ[A], Succ[B]] { type Result = Succ[Succ[S]] }
+ def apply[A <: Nat, B <: Nat](implicit plus: +[A, B]): Plus[A, B, plus.Result] = plus
+ }
+
+ val zero: +[_0, _0] = +.apply
+ val two: +[_0, _2] = +.apply
+ val four: +[_1, _3] = +.apply
+ // val invalidFour: +[_2, _3, _4] = +.apply
+ /*
+ - I need an implicit +[1, 3, 4] == +[Succ[0], Succ[2], Succ[Succ[2]]]
+ - can run inductive, but I ned an implicit +[0, 2, 2]
+ - can run basicRight and construct a +[0, 2, 2]
+ */
+```
+
+With that, we're ready to get started into our most complex work yet: sorting lists at compile time.
+
+## Lists as types
+
+As is now pretty standard for the series, we'll represent everything as a type. In our case, we'll represent lists of number-types as types again:
+
+```scala
+trait HList
+class HNil extends HList
+class ::[H <: Nat, T <: HList] extends HList
+```
+
+You may have seen similar definitions for heterogeneous lists (hence the HList name) in Shapeless and other libraries. In our case, our HList types are only restricted to Nat types.
+
+We're going to sort these HList types into other HList types, all inferred by the compiler. For that, we're going to use a merge-sort algorithm. You must know it already. We need to
+
+ - split the lists in half
+ - sort the halves
+ - then merge the halves back in a sorted order
+
+Every operation will be encoded as a type, and all results will be computed by the compiler via implicits.
+
+## Operation 1: The Split
+
+We need to be able to split a list in exactly half or at most one element difference (if the list has an odd number of elements). I bet you're used to encoding operations as types - this is what the second part's arithmetic was about - so we'll encode this as a type as well:
+
+```scala
+trait Split[HL <: HList, L <: HList, R <: HList]
+```
+
+In this case, HL is the original list, and L and R are the "left" and "right" halves of the list, respectively. We're going to make the compiler compute instances of this Split type, and thus prove that a list can be halved. As before, we're going to do everything in a companion of Split, starting with the basic case: an empty list is halved into two empty lists:
+
+```scala
+object Split {
+ implicit val basic: Split[HNil, HNil, HNil] = new Split[HNil, HNil, HNil] {}
+}
+```
+
+This is one of the starting points. Another basic case is that any one-element list can also be split into itself on the left, and empty on the right:
+
+```scala
+implicit def basic2[N <: Nat]: Split[N :: HNil, N :: HNil, HNil] =
+ new Split[N :: HNil, N :: HNil, HNil] {}
+```
+
+In the above, we're using the infix :: to make the types easier to read. So for any type N which is a "number", the compiler can automatically create an instance of `Split[N :: HNil, N :: HNil, HNil]`, which is proof of the existence of a split of a list of one element into itself and the empty type.
+
+The general inductive case, is when you have a list with at least two elements. That will have the type `N1 :: N2 :: T`, where T is some other list type (the tail of the list).
+
+```scala
+implicit def inductive[H <: Nat, HH <: Nat, T <: HList, L <: HList, R <: HList]
+ (implicit split: Split[T, L, R])
+ : Split[H :: HH :: T, H :: L, HH :: R]
+ = new Split[H :: HH :: T, H :: L, HH :: R] {}
+```
+
+In other words, if the tail T can be split into L and R - as detected by the compiler in the presence of an implicit `Split[T,L,R]` then N1 :: N2 :: T can be split into N1 :: L and N2 :: R. One number goes to the left, the other to the right.
+
+We can now add an apply method in Split:
+
+```scala
+def apply[HL <: HList, L <: HList, R <: HList](implicit split: Split[HL, L, R]) = split
+```
+
+and then test it out:
+
+```scala
+val validSplit: Split[_1 :: _2 :: _3 :: HNil, _1 :: _3 :: HNil, _2 :: HNil] = Split.apply
+```
+
+This works, because the compiler does the following:
+ - it requires an implicit `Split[_1 :: _2 :: _3 :: HNil, _1 :: _3 :: HNil, _2 :: HNil]`
+ - it can build that implicit by running inductive, but it needs an implicit `Split[_3 :: HNil, _3 :: HNil, HNil]`
+ - it can build that implicit by running `basic2[_3]`
+ - it will then build the dependent implicits as required
+
+Conversely, the compiler will not compile your code if the split is invalid:
+
+```scala
+// will not compile
+val invalidSplit: Split[_1 :: _2 :: _3 :: HNil, _1 :: HNil, _2 :: _3 :: HNil] = Split.apply
+```
+
+Though technically viable, the compiler needs to have a single proof for a split, so we chose the approach of "one number to the left, one to the right" and consider everything else invalid.
+
+## Operation 2: The Merge
+
+You know the drill - we'll create a new type which will have the meaning of a sorted merge of two lists:
+
+```scala
+trait Merge[LA <: HList, LB <: HList, L <: HList]
+```
+
+This means list LA merges with list LB and results in the final list L. We have two basic axioms we need to start with, and that is any list merged with HNil results in that list:
+
+```scala
+object Merge {
+ implicit def basicLeft[L <: HList]: Merge[HNil, L, L] =
+ new Merge[HNil, L, L] {}
+ implicit def basicRight[L <: HList]: Merge[L, HNil, L] =
+ new Merge[L, HNil, L] {}
+}
+```
+
+This time we need two basic axioms because the types `Merge[HNil, L, L]` and `Merge[L, HNil, L]` are different to the compiler.
+
+The inductive implicits are interesting. Considering two lists with at least an element each, say `HA :: TA` and `HB :: TB`, we need to compare their heads HA and HB:
+
+ - if HA <= HB, then HA must stay first in the result
+ - if HB < HA, then HB must stay first in the result
+
+The question is, what's the result?
+
+ - if HA <= HB, then the compiler must find a merge between TA and the other list `HB :: TB`, so it'll need an implicit instance of `Merge[TA, HB :: TB, O]`, where O is some HList, and the final result will be `HA :: O`
+ - if HB < HA, it's the other way around - the compiler needs to find an implicit of `Merge[HA :: TA, TB, O]` and then the final result will be `HB :: O`
+
+So we need to embed those rules as implicits:
+
+```scala
+implicit def inductiveLTE[HA <: Nat, TA <: HList, HB <: Nat, TB <: HList, O <: HList]
+ (implicit merged: Merge[TA, HB :: TB, O], lte: HA <= HB)
+ : Merge[HA :: TA, HB :: TB, HA :: O]
+ = new Merge[HA :: TA, HB :: TB, HA :: O] {}
+
+implicit def inductiveGT[HA <: Nat, TA <: HList, HB <: Nat, TB <: HList, O <: HList]
+ (implicit merged: Merge[HA :: TA, TB, O], g: HB < HA)
+ : Merge[HA :: TA, HB :: TB, HB :: O]
+ = new Merge[HA :: TA, HB :: TB, HB :: O] {}
+```
+
+Let's take the first case and read it: if the compiler can find an implicit `Merge[TA, HB :: TB, O]` and an implicit instance of `HA <= HB` (based on part 1), then the compiler will be able to automatically create an instance of `Merge[HA :: TA, HB :: TB, HA :: O]`, which means that HA stays at the front of the result.
+
+The other case reads in the exact same way except the conditions are the opposite: HB < HA, so HB needs to stay at the front of the result.
+
+Finally, if we add an apply method to Merge:
+
+```scala
+def apply[LA <: HList, LB <: HList, O <: HList](implicit merged: Merge[LA, LB, O]) = merged
+```
+
+then we should be able to test it:
+
+```scala
+val validMerge: Merge[_1 :: _3 :: HNil, _2 :: HNil, _1 :: _2 :: _3 :: HNil] = Merge.apply
+```
+
+This works, because the compiler
+
+ - requires an implicit `Merge[_1 :: _3 :: HNil, _2 :: HNil, _1 :: _2 :: _3 :: HNil]`
+ - will run the inductiveLTE, requiring an implicit `Merge[_3 :: HNil, _2 :: HNil, _2 :: _3 :: HNil]` and an implicit `_1 < _2`, which we'll assume true by virtue of Part 1
+ - will run inductiveGT, requiring an implicit `Merge[_3 :: HNil, HNil, _3 :: HNil]`
+ - will run basicLeft, creating an implicit `Merge[_3 :: HNil, HNil, _3 :: HNil]`
+ - will create all the dependent implicits in reverse order
+
+Conversely, if you try an invalid merge, the compiler won't compile your code because it can't find the appropriate implicits.
+
+## Operation 3: The Sort
+
+By now you should be ahead of my writing - encode the sort operation as a type:
+
+```scala
+trait Sort[L <: HList, O <: HList]
+```
+
+where L is the input list and O is the output list. Let's now think of the sorting axioms, again in the companion object of Sorted.
+
+The first basic axiom is that an empty list should stay unchanged:
+
+```scala
+implicit val basicNil: Sorted[HNil, HNil] = new Sorted[HNil, HNil] {}
+```
+
+Same for a list of one element:
+
+```scala
+implicit def basicOne[H <: Nat]: Sorted[H :: HNil, H :: HNil] =
+ new Sorted[H :: HNil, H :: HNil] {}
+```
+
+Now the inductive axiom is the killer one, as it will require all our previous work. We'll need the compiler to split the list, sort the halves and merge them back. Here's how we can encode that:
+
+```scala
+implicit def inductive[I <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList]
+ (implicit
+ split: Split[I, L, R],
+ sl: Sort[L, SL],
+ sr: Sort[R, SR],
+ merged: Merge[SL, SR, O])
+ : Sort[I, O]
+ = new Sort[I, O] {}
+```
+
+This reads as:
+ - given a Split of the input list into left (L) and right (R)
+ - given an existing instance of `Sort[L, SL]`
+ - given an existing instance of `Sort[R, SR]`
+ - given a Merge of SL and SR (which are sorted) into O
+
+then the compiler can automatically build an instance of `Sort[I, O]`. With a little bit of practice, this reads like natural language, doesn't it? Split, sort left, sort right, merge.
+
+Let's stick an apply method there:
+
+```scala
+def apply[L <: HList, O <: HList](implicit sorted: Sorted[L, O]) = sorted
+```
+
+And we should be free to test!
+
+```scala
+val validSort: Sort[
+ _4 :: _3 :: _5 :: _1 :: _2 :: HNil,
+ _1 :: _2 :: _3 :: _4 :: _5 :: HNil
+ ] = Sort.apply
+```
+
+This compiles, because the compiler will bend over backwards to try to find an implicit for every operation we need: the split of `_4 :: _3 :: _5 :: _1 :: _2 :: HNil`, the sorting of the smaller lists - which involve another splitting - and the merge of `_2 :: _4 :: _5 :: HNil` with `_1 :: _3 :: HNil` into the final result.
+
+Just plain amazing. The Scala compiler is awesome.
+
+## Operation Figure It Out
+
+We can make the compiler awesomer. What we have above is just asking for the compiler to validate whether sorting 4,3,5,1,2 results in 1,2,3,4,5. We can go further and make the compiler figure that out by itself.
+
+For this, we'll use the trick we used in Part 2, when we made the compiler figure out what the sum of two numbers was supposed to be. We'll do the same here.
+
+Instead of having the Sort type take two arguments, let's have it take only one and store the result as a type member instead:
+
+```scala
+trait Sort[L <: HList] {
+ type Result <: HList
+}
+```
+
+And in the Sort companion object, we'll create an auxiliary type with two type arguments:
+
+```scala
+type SortOp[L <: HList, O <: HList] = Sort[L] {
+ type Result = O
+ }
+```
+
+Now with this SortOp type alias, we'll make our implicits return this SortOp type, but since we can't instantiate it by itself, we'll need to create a Sort instance with the right type member:
+
+```scala
+implicit val basicNil: SortOp[HNil, HNil] =
+ new Sort[HNil] { type Result = HNil }
+```
+
+The signature looks identical to what we had before (we're returning an instance with two type arguments), but in the implementation we're returning an instance of Sort with a single type argument and the right type member inside. This is the trick we'll use to make the compiler figure out the result type for us.
+
+A similar implementation for the other basic axiom:
+
+```scala
+implicit def basicOne[H <: Nat]: SortOp[H :: HNil, H :: HNil] =
+ new Sort[H :: HNil] { type Result = H :: HNil }
+```
+
+And a similar approach (with changes in multiple places for the inductive implicit:
+
+```scala
+implicit def inductive[I <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList]
+ (implicit
+ split: Split[I, L, R],
+ sl: SortOp[L, SL],
+ sr: SortOp[R, SR],
+ merged: Merge[SL, SR, O])
+ : SortOp[I, O]
+ = new Sort[I] { type Result = O }
+```
+
+And with that, we can make the compiler figure out what the result type should be:
+
+```scala
+> show(Sort[_4 :: _3 :: _5 :: _1 :: _2 :: HNil])
+TypeTag[... { type Result =
+ Succ[_0]
+ :: Succ[Succ[_0]]
+ :: Succ[Succ[Succ[_0]]]
+ :: Succ[Succ[Succ[Succ[_0]]]]
+ :: Succ[Succ[Succ[Succ[Succ[_0]]]]]
+ :: HNil
+}]
+```
+
+In other words, the result is 1, 2, 3, 4, 5.
+
+We love you, compiler.
+
+## The Final Code
+
+This is the final version of what we wrote in this part. Stick the first two parts before it and you'll have the greatest merge sort in Scala the world has ever seen.
+
+```scala
+trait HList
+class HNil extends HList
+class ::[H <: Nat, T <: HList] extends HList
+trait Split[HL <: HList, L <: HList, R <: HList]
+object Split {
+ implicit def basic: Split[HNil, HNil, HNil] = new Split[HNil, HNil, HNil] {}
+ implicit def basic2[H <: Nat]: Split[H :: HNil, H :: HNil, HNil] = new Split[H :: HNil, H :: HNil, HNil] {}
+ implicit def inductive[H <: Nat, HH <: Nat, T <: HList, L <: HList, R <: HList] (implicit split: Split[T, L, R]): Split[H :: HH :: T, H :: L, HH :: R] =
+ new Split[H :: HH :: T, H :: L, HH :: R] {}
+ def apply[HL <: HList, L <: HList, R <: HList](implicit split: Split[HL, L, R]) = split
+}
+
+val validSplit: Split[_1 :: _2 :: _3 :: HNil, _1 :: _3 :: HNil, _2 :: HNil] = Split.apply // good
+// val invalidSplit: Split[_1 :: _2 :: _3 :: HNil, _1 :: _2 :: HNil, _3 :: HNil] = Split.apply // doesn't compile
+
+trait Merge[LA <: HList, LB <: HList, L <: HList]
+object Merge {
+ implicit def basicLeft[L <: HList]: Merge[HNil, L, L] = new Merge[HNil, L, L] {}
+ implicit def basicRight[L <: HList]: Merge[L, HNil, L] = new Merge[L, HNil, L] {}
+ implicit def inductiveLTE[HA <: Nat, TA <: HList, HB <: Nat, TB <: HList, O <: HList] (implicit merged: Merge[TA, HB :: TB, O], lte: HA <= HB)
+ : Merge[HA :: TA, HB :: TB, HA :: O] = new Merge[HA :: TA, HB :: TB, HA :: O] {}
+ implicit def inductiveGT[HA <: Nat, TA <: HList, HB <: Nat, TB <: HList, O <: HList] (implicit merged: Merge[HA :: TA, TB, O], g: HB < HA)
+ : Merge[HA :: TA, HB :: TB, HB :: O] = new Merge[HA :: TA, HB :: TB, HB :: O] {}
+ def apply[LA <: HList, LB <: HList, O <: HList](implicit merged: Merge[LA, LB, O]) = merged
+}
+
+val validMerge: Merge[_1 :: _3 :: HNil, _2 :: HNil, _1 :: _2 :: _3 :: HNil] = Merge.apply // compiles
+
+trait Sort[L <: HList] {
+ type Result <: HList
+}
+object Sort {
+ type SortOp[L <: HList, O <: HList] = Sort[L] { type Result = O }
+ implicit val basicNil: SortOp[HNil, HNil] = new Sort[HNil] { type Result = HNil }
+ implicit def basicOne[H <: Nat]: SortOp[H :: HNil, H :: HNil] = new Sort[H :: HNil] { type Result = H :: HNil }
+ implicit def inductive[I <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList]
+ (implicit
+ split: Split[I, L, R],
+ sl: SortOp[L, SL],
+ sr: SortOp[R, SR],
+ merged: Merge[SL, SR, O])
+ : SortOp[I, O]
+ = new Sort[I] { type Result = O }
+
+ def apply[L <: HList](implicit sorted: Sort[L]): SortOp[L, sorted.Result] = sorted
+}
+
+println(show(Sort[_4 :: _3 :: _5 :: _1 :: _2 :: HNil])) // if it compiles, you're good!
+```
diff --git a/_posts/2020-08-29-eta-expansion-and-paf.md b/_posts/2020-08-29-eta-expansion-and-paf.md
new file mode 100644
index 000000000000..2d7d9e7cd186
--- /dev/null
+++ b/_posts/2020-08-29-eta-expansion-and-paf.md
@@ -0,0 +1,133 @@
+---
+title: "Eta-Expansion and Partially Applied Functions in Scala"
+date: 2020-08-29
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, functional programming]
+excerpt: "We explore some not-so-well-understood territory in the form of eta-expansion and how methods and functions interoperate."
+---
+This article is for Scala programmers who know at least these essential concepts: what a method is and how to define a function value (lambda). Here we'll discuss the topic of eta-expansion and partially-applied functions, which are often loosely covered and piecemeal.
+
+## Background
+
+Long story short, methods and functions are different things in Scala. When I write
+
+```scala
+def incrementMethod(x: Int): Int = x + 1
+```
+
+that's a method, which is a piece of code that can be invoked on an instance of a class. A method is a member of the enclosing class or object and can be invoked like this:
+
+```scala
+val three = someInstance.incrementMethod(2)
+```
+
+Even if you call the method from the body of the class or object, it's the same as invoking it on the `this` instance. So you won't be able to call a method on its own, because it's tied to some instance of a class.
+
+Function values (aka lambdas), on the other hand, are pieces of code that can be invoked independently of a class or object. Functions are assignable to values or variables, can be passed as arguments and can be returned as results - it's one of the most important tenets of functional programming in Scala.
+
+```scala
+val incrementFunction = (x: Int) => x + 1
+val three = incrementFunction(2)
+```
+
+Behind the scenes, these function values are actually instances of the FunctionN family of traits with an apply method which benefits from special treatment, so what you're doing is in fact:
+
+```scala
+// what the compiler does
+val incrementFunction = new Function1[Int, Int] {
+ override def apply(x: Int): Int = x + 1
+ }
+val three = incrementFunction.apply(2) // desugared from incrementFunction(2)
+```
+
+Methods and functions are thus different in Scala. However, because the user sees and uses them in the same way (just invoke them), they're "morally" equivalent. The eta-expansion mechanism allows the conversion between a method and a function.
+
+## Converting a Method to a Function
+
+Because a method and a function are seen differently by the JVM - a method of a class vs a field of type FunctionN - you can't simply say
+
+```scala
+val incrementF = incrementMethod
+```
+
+because the compiler will think you'll try to call your increment method, which requires arguments. The way you'd do the conversion is
+
+```scala
+val incrementF = incrementMethod _
+```
+
+The underscore at the end is a signal for the compiler that you want to turn the method into a function value, and you'll obtain a function of type `Int => Int`. This conversion is called _eta-expansion_, and the compiler will generate a piece of code that will look something like
+
+```scala
+val incrementF = (x: Int) => incrementMethod(x)
+```
+
+The compiler can also do this automatically if you give it the function type in advance:
+
+```scala
+val incrementF2: Int => Int = incrementMethod
+```
+
+In this case, the compiler can disambiguate the context, because you declared that you want a function so the compiler will automatically eta-expand the method for you.
+
+## Partially Applied Functions
+
+Another important scenario where eta-expansion is useful is with methods taking multiple argument lists.
+
+```scala
+def multiArgAdder(x: Int)(y: Int) = x + y
+val add2 = multiArgAdder(2) _
+```
+
+In this case, you'll get another function which takes the remaining arguments, therefore of type `Int => Int`. This is called a _partially applied function_, because you're only supplying a subset of argument lists and the remaining arguments are to be passed later:
+
+```scala
+val three = add2(1)
+```
+
+In a similar fashion as before, the compiler can detect whether a value is expected to have a function type, and so it can automatically eta-expand a method for you:
+
+```scala
+List(1,2,3).map(multiArgAdder(3)) // eta-expansion is done automatically
+```
+
+In this case, the argument of the map method needs to have the type `Int => Int`, so the compiler will automatically turn the method into an eta-expanded lambda for you.
+
+## Interesting Questions
+
+So far, we've discussed only methods that have a single argument in their list. Here's something to think about:
+
+```scala
+def add(x: Int, y: Int) = x + y
+val addF = add _
+```
+
+In this case, the method has two arguments. An eta-expanded function value (lambda) will have two arguments as well, so it will be of type `(Int, Int) => Int`. A similar expansion will happen on a larger number of arguments as well.
+
+Another interesting scenario is: what happens on more than two argument lists and/or we're left with more than an argument list in the expanded function:
+
+```scala
+def threeArgAdder(x: Int)(y: Int)(z: Int) = x + y + z
+val twoArgsRemaining = threeArgAdder(2) _
+```
+
+In this case, we'll get a curried function which will take the remaining argument lists in turn, so the function type will be `Int => Int => Int`. If you want to invoke it:
+
+```scala
+val ten = twoArgsRemaining(3)(5)
+```
+
+At the same time, if we pass more than one argument list:
+
+```scala
+val oneArgRemaining = threeArgAdder(2)(3) _
+```
+
+then we'll get a function which takes the remaining argument lists (a single integer), so it will have the type `Int => Int`.
+
+_In general and to put it short, eta-expansion turns a method into a function which will take the remaining argument lists (however large) in turn, however long the chain may be._
+
+## Conclusion
+
+Eta-expansion in Scala is often covered in small pieces in StackOverflow questions, so I hope this small blog will paint a clearer and more general picture of what it is and why it's useful in Scala code.
diff --git a/_posts/2020-08-31-repartition-coalesce.md b/_posts/2020-08-31-repartition-coalesce.md
new file mode 100644
index 000000000000..a8b38bcf65f1
--- /dev/null
+++ b/_posts/2020-08-31-repartition-coalesce.md
@@ -0,0 +1,112 @@
+---
+title: "Repartition vs Coalesce in Apache Spark"
+date: 2020-08-31
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [spark]
+excerpt: "Drawing the line between two fundamental repartitioning operations in Apache Spark."
+---
+This article is for the Scala & Spark programmers, particularly those Spark programmers that are starting to dive a little deeper into how Spark works and perhaps attempting to make it faster. I'm expecting you to be at least familiar with:
+
+ - the distributed nature of Spark
+ - the concept of partition as a chunk of data in an RDD or DataFrame
+
+In this article, I'm going to discuss the commonalities and the differences between `repartition` and `coalesce`.
+
+## The Boilerplate
+
+I'm going to work in an IntelliJ project with the Spark library installed, so add the following to your build.sbt file:
+
+```scala
+libraryDependencies += "org.apache.spark" %% "spark-core" % "3.0.0"
+```
+
+I'm also going to work in a standalone application, for which I'm going to spin up a dedicated Spark Session:
+
+```scala
+object RepartitionCoalesce {
+
+ val spark = SparkSession.builder()
+ .appName("Repartition and Coalesce")
+ .master("local[*]")
+ .getOrCreate()
+
+ val sc = spark.sparkContext
+}
+```
+
+You're probably aware of what this code does - the SparkSession is the entry point for the high-level DataFrame API, and the SparkContext is the entry point for the low-level RDD API.
+
+The techniques I'm going to show in this article are 100% applicable to both RDDs and DataFrames, but to demonstrate their differences, I'm going to generate more data more quickly with the RDD API.
+
+```scala
+val numbers = sc.parallelize(1 to 10000000)
+println(numbers.partitions.length)
+```
+
+When you run this application, provided that you configured the SparkSession with `local[*]` master (like in the snippet), you'll most likely see the number of your virtual cores. By default, this is how an RDD will be split. This demo will be most visible if you have more than 2 virtual cores on your computer.
+
+## Repartition
+
+Repartition is a method which is available for both RDDs and DataFrames. It allows you to change the number of partitions they are split into. This is particularly useful when your partitions are very small and data processing is slow because of it - you can hit repartition with a smaller number of partitions, and your data will be redistributed in between them. Try to repartition your data into fewer partitions than the number of cores:
+
+```scala
+val repartitionedNumbers = numbers.repartition(2)
+repartitionedNumbers.count()
+```
+
+A repartition will incur a shuffle - a data exchange between executors in the cluster. When we deal with Spark performance, we generally want to avoid shuffles, but if it's for a good cause, it might be worth it. Regardless of whether it's worth it or not, repartition involves a shuffle - this is the important bit.
+
+## Coalesce
+
+Another method for changing the number of partitions of an RDD or DataFrame is `coalesce`. It has a very similar API - just pass a number of desired partitions:
+
+```scala
+val coalescedNumbers = numbers.coalesce(2)
+coalescedNumbers.count()
+```
+
+## The Test
+
+Let's run a small test - execute this application with the count action executed on the two RDDs repartitioned and coalesced to 2 partitions, respectively. If you need to, add a sleep in the main thread so you can watch the Spark UI. It'll take no more than a few seconds to run. Since we have two `count` actions there, we'll have two jobs running.
+
+If you look at the Spark UI, you'll see something very interesting:
+
+
+
+The first job (repartition) took 3 seconds, whereas the second job (coalesce) took 0.1 seconds! Our data contains 10 million records, so it's significant enough.
+
+There must be something fundamentally different between repartition and coalesce.
+
+## The Difference
+
+We can explain what's happening if we look at the stage/task decomposition of both jobs. In this case, the DAGs are very useful. Let's look at repartition:
+
+
+
+In other words, we have two stages and a shuffle in between them. As I mentioned earlier, this is expected - a repartition will redistribute the data evenly between the new number of partitions, so a shuffle is involved. As you probably know, shuffles are expensive, even for the ~70MB of data we're dealing with.
+
+Let's look at the other DAG of the coalesce job:
+
+
+
+In other words, a single stage! _Coalesce does not involve a shuffle_. Why doesn't it incur a shuffle since it changes the number of partitions?
+
+Coalesce changes the number of partitions in a fundamentally different way. Instead of redistributing the data evenly between the new number of partitions, coalesce will simply "stitch" partitions together, so there's no data movement between all executors, but only between those involved in the respective partitions.
+
+Put in a different way: in the case of a repartition, each input partition will spread data into _all_ output partitions. In the case of coalesce, each input partition is included in _exactly one_ output partition. This causes massive performance improvements in the case of coalesce, _when you're decreasing the number of partitions._ If you're increasing the number of partitions, a coalesce is identical to a repartition.
+
+So why ever do a repartition at all?
+
+The fundamental benefit of a repartition is _uniform data distribution_, which a coalesce can't guarantee - since it stitches partitions together, the output partitions might be of uneven size, which may or may not cause problems later, depending on how skewed they are.
+
+## Conclusion
+
+With bullets,
+
+ - repartition redistributes the data evenly, but at the cost of a shuffle
+ - coalesce works much faster when you reduce the number of partitions because it sticks input partitions together
+ - coalesce doesn't guarantee uniform data distribution
+ - coalesce is identical to a repartition when you increase the number of partitions
+
+And with that, massive performance benefits await if you know how to make the right tradeoffs, as with anything in life.
diff --git a/_posts/2020-09-05-underscore-in-scala.md b/_posts/2020-09-05-underscore-in-scala.md
new file mode 100644
index 000000000000..a3d9dbb457fb
--- /dev/null
+++ b/_posts/2020-09-05-underscore-in-scala.md
@@ -0,0 +1,138 @@
+---
+title: "Underscores are Overloaded in Scala!"
+date: 2020-09-05
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala]
+excerpt: "Scala syntax is so confusing sometimes - I'll show almost all uses of underscores in Scala. Sometimes the philosophy is inconsistent, but it's worth at least being aware."
+---
+This article is for the curious Scala programmer who has seen underscores more times than they'd find comfortable. In this article I'll share almost all the places where you might see an underscore, so that you don't freak out when you see yet another one in a different context.
+
+I want to get straight to the point.
+
+## 1. Ignoring
+
+Underscores are commonly used when names or values are not important and/or unused. Here is a comprehensive list of ignored elements. First, variable names:
+
+```scala
+val _ = 5
+```
+
+Why you'd do that, I have no idea - the point of intentionally declaring a value or variable is to use it. But it's possible. Probably the most useless case of an underscore. However, the case when the compiler throws a value at use, but we don't use it, so we use an underscore. The most common use case is with lambdas whose arguments we don't need or use:
+
+```scala
+val onlyFives = (1 to 10).map(_ => 5)
+```
+
+We could have said `(1 to 10).map(x => 5)` but we don't need x and the compiler would have issued an unused variable warning anyway, so we replace with an underscore.
+
+Another scenario is when we use self-types as a type restriction, but you don't actually need to name the self-type:
+
+```scala
+trait Singer
+trait Actor { _: Singer =>
+ // implementation
+}
+```
+
+Finally, one of the less common places where underscores ignore stuff is generic types. For example:
+
+```scala
+def processList(list: List[Option[_]]): Int = list.length
+```
+
+Since we don't care what type those Options are, we type them with underscore. Underscores are also useful when we want to interoperate with Java libraries or generic types which were not explicitly typed (think pre-Java 5).
+
+## 2. Wildcards
+
+Underscores are also used to have the meaning of "everything" in certain contexts. One of them is importing everything in a package:
+
+```scala
+import cats.implicits._
+```
+
+But by far the most frequent use case is in pattern matching, where the underscore means "match everything".
+
+```scala
+meaningOfLife match {
+ case _ => "I'm fine with anything"
+}
+```
+
+## 3. Default Initializers
+
+Particularly for variables, when we don't know what to initialize them with, let the JVM decide - zero for numerical values, false for Booleans, null for reference types:
+
+```scala
+var myVariable: String = _
+```
+
+## 4. Lambda Sugars
+
+You've probably seen this before. Let's say we want to multiply every element of a list by 5, we would do
+
+```scala
+List(1,2,3,4).map(x => x * 5)
+```
+
+Except there's an even shorter version. We can wrote
+
+```scala
+List(1,2,3,4).map(_ * 5)
+```
+
+which means the same thing: the compiler expands that to the slightly longer lambda version. The downside is that we can only use the underscore once in the function body - that's because each underscore represents a different argument of the lambda. For example, if I wrote
+
+```scala
+val sumFunction: (Int, Int) => Int = _ + _
+```
+
+The `_ + _` part is identical to `(a, b) => a + b`. Java programmers seeing this for the first time might find it too short, but it's really easy to get used to it.
+
+## 5. Eta-expansion
+
+Underscores are also used to turn methods into function values. We talk about the process in detail in another article (check the link above).
+
+```scala
+def incrementer(x: Int) = x + 1
+val incrementerFunction = incrementer _
+```
+
+The underscore is a signal to the compiler to create a new lambda with the implementation of `x => incrementer(x)`.
+
+## 6. Higher-Kinded Types
+
+HKTs are generic types, whose type arguments are themselves generic. Libraries like Cats exploit this like crazy. Their structure is of the form
+
+```scala
+class MyHigherKindedJewel[M[_]]
+```
+
+Where the type argument M is also generic. If I want to instantiate my HKT with a concrete generic type like List, I'll use the List type (not a List[String] or something else):
+
+```scala
+val myJewel = new MyHigherKindedJewel[List]
+```
+
+## 7. Vararg methods
+
+When we have a method taking a variable number of arguments, such as
+
+```scala
+def makeSentence(words: String*) = ...
+```
+
+we might find ourselves we need to expand some collection of Strings as arguments to this method. We can't pass them manually at runtime, so we need to expand the collection automatically:
+
+```scala
+val words = List("I", "love", "Scala")
+val love = makeSentence(words: _*)
+```
+
+In this way, the elements of the list are exploded into the argument list to the method.
+
+## Conclusion
+
+Scala overloaded the meaning of the underscore in many, many places - hopefully this article served as a good overview of where you might see it in real code.
+
+Let me know if I've missed anything!
diff --git a/_posts/2020-09-09-refined-types.md b/_posts/2020-09-09-refined-types.md
new file mode 100644
index 000000000000..2cde7dcc8ff4
--- /dev/null
+++ b/_posts/2020-09-09-refined-types.md
@@ -0,0 +1,160 @@
+---
+title: "Refined Types in Scala"
+date: 2020-09-09
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system, refined]
+excerpt: "We look at how we can impose constraints on values at compile time with the Refined library."
+---
+This article is for Scala programmers of all levels, although if you're a more advanced programmer or you watched/read the type-level programming mini-series , you will get more value out of it because you'll have a good intuition for what happens behind the scenes.
+
+I'm talking about "behind the scenes" because today's topic involves lots of Scala magic: refined types.
+
+## Introduction
+
+So what's the problem?
+
+We often work with data that cause us problems: negative numbers, empty strings, emails that don't contain "@", and so on. To handle them, we usually write a lot of defensive code so that our application doesn't crash at runtime.
+
+The problem is that very often, such problems can be avoided altogether before the application is compiled. Say we're writing a distributed chat application, and our users are signed in by email:
+
+```scala
+case class User(name: String, email: String)
+```
+
+In this case, we're going to be perfectly fine citizens and write
+
+```scala
+val daniel = User("Daniel", "daniel@rockthejvm.com")
+```
+
+But when someone else created that type and we want to create an instance of User, we can be sloppy, forgetful or simply innocent and write
+
+```scala
+val noDaniel = User("daniel@rockthejvm.com", "Daniel")
+```
+
+and it would be perfectly fine to the compiler... until the application crashes at some point because you're attempting to send an email to "Daniel".
+
+## Enter Refined
+
+If you want to write code with me in this article, I'll invite you to add the following to your build.sbt file:
+
+```scala
+libraryDependencies += "eu.timepit" %% "refined" % "0.9.15"
+```
+
+(of course, if you want a newer version, check the library's releases page on GitHub )
+
+Refined is a small library for Scala that wants to help us avoid this kind of preventable crashes at compile time. We can leverage the power of the Scala compiler to validate certain predicates before our application has the chance to run and ruin our day. Let's start with something simple. Say we're using only positive numbers in our application, but there's no such thing as a positive integer type. We can use a refined type over Int, to enforce that the number is positive:
+
+```scala
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.auto._
+import eu.timepit.refined.numeric._
+
+val aPositiveInteger: Refined[Int, Positive] = 42
+```
+
+Refined is a type that takes two type arguments: the "value" type you're testing/validating, and the predicate you want the value to pass. If you've gone through the type-level programming mini-series earlier, you have a good warmup: the predicate is embedded as a type. Obviously, there's some implicit conversion happening behind the scenes so you can simply write 42 on the right-hand side.
+
+Refined is pretty powerful, because if you use a value that's not appropriate
+
+```scala
+val aNegativeInteger: Refined[Int, Positive] = -100
+```
+
+the code will not even compile! This is the main benefit of Refined: it helps the compiler catch your errors before you even deploy your application.
+
+## Refined Tools
+
+Now that you're more familiar with the problem and how Refined can solve it, let's go over some of the capabilities of the library. I'm going to go over the most useful of them.
+
+You saw the numerical Positive predicate. There are tons of others:
+
+ - allow only negative numbers
+ - allow only non-negative numbers (including 0)
+ - allow only odd numbers
+ - allow only even numbers
+
+```scala
+val aNegative: Int Refined Negative = -100
+val nonNegative: Int Refined NonNegative = 0
+val anOdd: Int Refined Odd = 3
+val anEven: Int Refined Even = 68
+```
+
+Notice I used Refined in infix notation: `Refined[Int, Odd]` can also be written as `Int Refined Odd`.
+
+There are also some more interesting filters. For example, allow only numbers less than a certain value. This is possible at compile time with the magic provided by shapeless and its macros:
+
+```scala
+import eu.timepit.refined.W
+val smallEnough: Int Refined LessThan[W.`100`.T] = 45
+```
+
+The `W` value is an alias for shapeless' Witness, which is able to generate a type for us with the construct `W.`100`.T`. Whenever you need to create one of these types yourself, you would use a construct like that.
+
+With this new tools, a whole lot of other functionalities for filtering numbers are unlocked:
+
+ - less than a certain number (or less-than-or-equal)
+ - greater than a certain number (or gte)
+ - in an open/closed interval between numbers
+ - divisible by a number
+ - whose modulo is a certain number
+
+Again, all available at compile time!
+
+## Refined Tools, Supercharged
+
+However, the most useful validations happen on strings. Since so much of our application logic is dependent on strings, it makes sense to want to validate them in many ways. For strings, by far the most useful filters are ends-with, starts-with and regex matching:
+
+```scala
+import eu.timepit.refined.string._
+
+val commandPrompt: String refined EndsWith[W.`"$"`.T] = "daniel@mbp $"
+```
+
+Regex is probably the most powerful - the library allows you to both test whether a string is a regex, and if a string matches a regex:
+
+```scala
+val isRegex: String Refined Regex = "rege(x(es)?|xps?)"
+```
+
+If the string you use is not a valid regex string, the compilation will fail. For regex matching:
+
+```scala
+type Email = String Refined MatchesRegex[W.`"""[a-z0-9]+@[a-z0-9]+\\.[a-z0-9]{2,}"""`.T]
+```
+
+In the above we use MatchesRegex and we use triple-quotes to not have to escape every backslash again. With MatchesRegex, you can go wild and add validations for everything you may want:
+
+```scala
+type SimpleName = String Refined MatchesRegex[W.`"""[A-Z][a-z]+"""`.T]
+case class ProperUser(name: SimpleName, email: Email)
+
+val daniel = ProperUser("Daniel", "daniel@rockthejvm.com")
+// val noDaniel = ProperUser("daniel@rockthejvm.com", "Daniel") // doesn't compile
+```
+
+## Refining at Runtime
+
+Granted, we can't work with manually-inserted literals all the time - probably not even most of the time. The Refined library allows you to put a value through a predicate and return an Either which contains the predicate failing error (as a String) or the refined type:
+
+```scala
+import eu.timepit.refined.api.RefType
+
+val poorEmail = "daniel"
+val refineCheck = RefType.applyRef[Email](poorEmail)
+```
+
+After you've done the check, you can pattern-match the result and move along with your parsed value.
+
+## Conclusion
+
+You've hopefully learned a new powerful tool to test values at compile time and catch nasty bugs before your application is even deployed. If you need some intuition on how Refined works, check out the type-level programming mini-series:
+
+ - part 1
+ - part 2
+ - part 3
+
diff --git a/_posts/2020-09-10-lambda.md b/_posts/2020-09-10-lambda.md
new file mode 100644
index 000000000000..ed092f90bd62
--- /dev/null
+++ b/_posts/2020-09-10-lambda.md
@@ -0,0 +1,84 @@
+---
+title: "How to Deploy a Scala Application to AWS Lambda"
+date: 2020-09-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, aws, how to]
+excerpt: "Scala code is super easy to deploy to AWS lambda. We show you how in this step by step tutorial."
+---
+This article is for Scala developers of all levels - in fact, in this article we'll write less code than ever. I'll demonstrate how to deploy a Scala application to AWS Lambda, step by step.
+
+## Introduction
+
+AWS Lambda needs little introduction. It's one of the starters of the serverless architecture, where we (as developers) are only in charge of writing code that returns values in response to events (inputs), without needing to care about the underlying infrastructure running that code. Naturally, this architecture has become very popular in recent years. It also matches the functional programming mindset very closely.
+
+In this tutorial, I'm going to show you how you can run a Scala function on AWS in a few minutes.
+
+## The Code and Building the Function
+
+Simply create a new Scala project with your favorite tool (I'll use IntelliJ). Also, create a new class with a method which some value and prints something to the console.
+
+```scala
+package demolambda
+
+class DemoLambda {
+
+ def execute(): Int = {
+ println("I'm running!")
+ 42
+ }
+}
+```
+
+After you've written the magical code, you'll need to package your application with your favorite build tool. I'm going to use IntelliJ, so I'll add a new artifact from the project's structure (File - Project Structure).
+
+
+
+Then make sure you add the right module and that the dependencies are included in the final jar (check the small radio button below):
+
+
+
+After you've created the artifact definition, you'll need to build it. Go to _Build -> Build artifacts_, select your new artifact, then Build. The compiler will take care to create your new JAR file.
+
+The JAR file will be under the _out/_ folder in your IntelliJ project. Remmeber that path, because you'll need to upload it to AWS. As a sanity check, look at the JAR's size: it should be a few MB.
+
+## Setting Up the Lambda Function
+
+Now for the fun stuff. Go to your AWS console . If you don't have an AWS account, go ahead and create one. Then select the Lambda function from the 572389057823 services Amazon offers you.
+
+
+
+Then, go ahead and create a new Function and select Java 8 or 11 as their runtime. Both will work fine for this demo.
+
+
+
+
+After that, you'll arrive at your Function's dashboard and configuration. Go ahead and upload the JAR you've built a minute ago:
+
+
+
+After that, you need to go down to the Basic Settings panel and Edit the lambda's entry point:
+
+
+
+Make sure you use the fully qualified class name of your Scala class, then use the Java notation to identify the method you want to run. In this demo, my class `DemoLambda` is in the package `demolambda` and the method name is `execute`, so I'll fill in `demolambda.DemoLambda::execute`.
+
+
+
+After you've done that, you'll need to set up a test run. Click the Test button at the top.
+
+
+
+You'll get a dialog with a standard test event as JSON. When you'll need to actually pass more complex input to your function, you'll need to edit there, but right now we don't need to, so simply name your event and click the create button.
+
+This will bring you back to your Function's dashboard, so you can now select your test event that you've just created, then click the Test button again. After a few seconds, you'll see a green box with the successful test run. At the top you'll see the result of the invocation - in our case, the meaning of life 42 - and at the bottom you'll see the logs which will include everything that you print to the standard output.
+
+
+
+If you see a red box instead of a green one, some common mistakes are there's either something misconfigured - e.g. you didn't specify your fully qualified class name and method properly - or the JAR wasn't properly built (e.g. it doesn't contain the Scala runtime libraries).
+
+## Conclusion
+
+You've learned how to deploy a Scala application to AWS Lambda. Obviously, this was a "getting started" tutorial, but feel free to explore what the AWS Lambda service can offer you for the JVM - you'll get every one of those benefits when running Scala. For AWS Lambda documentation with Java (including how to handle inputs) check this link and the docs around it:
+
+AWS Lambda Java (JVM) documentation
diff --git a/_posts/2020-09-16-akka-http-json.md b/_posts/2020-09-16-akka-http-json.md
new file mode 100644
index 000000000000..554c55bb9315
--- /dev/null
+++ b/_posts/2020-09-16-akka-http-json.md
@@ -0,0 +1,297 @@
+---
+title: "Akka HTTP loves JSON: 3 Libraries You Can Integrate into Akka HTTP"
+date: 2020-09-16
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka http, spray, circe, jackson]
+excerpt: "Akka HTTP needs JSON like humans need water. We show you how to integrate Spray-Json, Circe and Jackson into Akka HTTP."
+---
+This article is for the Scala programmer who works with Akka HTTP . Probably one of the most common problems for any developer writing HTTP services is, obviously, JSON manipulation.
+
+In this article we'll address not one, but 3 different ways to handle JSON in Akka HTTP.
+
+## The Background
+
+To work alongside this article, you'll need to add the Akka actors and Akka HTTP libraries to your build.sbt file:
+
+```scala
+val akkaVersion = "2.6.5"
+val akkaHttpVersion = "10.2.0"
+
+libraryDependencies ++= Seq(
+ "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion,
+ "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion,
+)
+```
+
+Because this article is focused exclusively on JSON manipulation, we'll steer away from the massive complexity (and power) of Akka HTTP directives and just focus on some of them: `get`/`post`, `entity` and `as`. We'll briefly go through them when we use them so you aren't left in the dark if you haven't seen them before.
+
+For this article, we'll create a simple HTTP service which exposes an HTTP POST endpoint. Assume we're working on a dating app, so a central data structure will be a person, containing their name and age. We intend to add such persons to an internal database, and return events that have a unique identifier and a timestamp:
+
+```scala
+case class Person(name: String, age: Int)
+case class UserAdded(id: String, timestamp: Long)
+```
+
+Now, let's assume we're working on the user creation service of this dating platform which exposes a POST endpoint which takes a Person object encoded as JSON, "adds" it to our "database", then replies back with a JSON string. We need to be able to (de)serialize between the JSON payload and our internal data structures.
+
+Here's a simple Akka HTTP server that does everything but the JSON part:
+
+```scala
+import akka.actor.typed.ActorSystem
+import akka.actor.typed.scaladsl.Behaviors
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server.Route
+
+/*
+ We're going to work with these data structures in our endpoints.
+ In practice we'd create some "request" data structures, e.g. AddUserRequest.
+*/
+case class Person(name: String, age: Int)
+case class UserAdded(id: String, timestamp: Long)
+
+object AkkaHttpJson {
+ implicit val system = ActorSystem(Behaviors.empty, "AkkaHttpJson")
+
+ val route: Route = (path("api" / "user") & post) {
+ complete("Yep, roger that!")
+ }
+
+ def main(args: Array[String]): Unit = {
+ Http().newServerAt("localhost", 8081).bind(route)
+ }
+}
+```
+
+Of course, this example is really simple: we accept any post request on the path "api/user", and we reply with 200 OK and the payload "Yep, roger that!".
+
+Let's make this server accept JSON payloads, first by using the go-to library for Akka HTTP which is also maintained by the Akka team: Spray-json.
+
+## 1. Spray-json
+
+To add Spray-json to your project, you'll need to add the following dependency to your build.sbt:
+
+```scala
+ // (inside the library dependencies sequence)
+ "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion
+```
+
+Now, to use Spray-json, you'll need to follow a few steps. JSON manipulation is not 100% automatic, but you won't need to write too much code either.
+
+Step 1. You'll need to add the Spray-json package to your application file, so that you can access the API.
+
+```scala
+import spray.json._
+```
+
+Step 2. You'll need to create a scope - a trait, an object etc - which has JSON converters for any type that you might want to support. Here, we have just the Person type, so we'll use this as an example.
+
+```scala
+trait PersonJsonProtocol extends DefaultJsonProtocol {
+ /*
+ The `jsonFormat2` method is applicable for Product types (which include case classes) with 2 members.
+ If you have case classes with more than 2 fields, you'd use the appropriate `jsonFormatX` method.
+ */
+ implicit val personFormat = jsonFormat2(Person)
+ implicit val userAddedFormat = jsonFormat2(UserAdded)
+}
+```
+
+Step 3. You'll need to add the implicit JSON converters into the scope of your route. If you created the converters within a trait, as we did above, you'll simply need to mix the trait into your main app. If you wrote them inside an object, you'll need to import the contents of that object. Besides this, you'll also need to mix-in the trait `SprayJsonSupport`, which contains some implicit definitions that the Akka HTTP server directives will need.
+
+```scala
+object AkkaHttpJson extends PersonJsonProtocol with SprayJsonSupport {
+ // ... the rest of the code
+}
+```
+
+Step 4. Add the `entity` and `as` server directives into the route, so that the payload from the HTTP request is automatically parsed and converted into the type that you wanted to support - in our case Person:
+
+```scala
+ // at the top
+ import java.util.UUID
+
+ // inside the server app
+ val route: Route = (path("api" / "user") & post) {
+ entity(as[Person]) { person =>
+ complete(UserAdded(UUID.randomUUID().toString, System.currentTimeMillis()))
+ }
+ }
+```
+
+The code is short but magical. Here's how it works in a nutshell:
+
+
+ - `as[Person]` fetches whatever implicit marshaller (serializer) of Persons the compiler has access to. Because you've imported the implicit JSON format for Person in scope (from step 3) and added SprayJsonSupport - which is able to convert between an Akka HTTP Entity and the internal JSON format of Spray-json - the compiler does the rest.
+ - The `entity` directive uses the marshaller that you pass as argument to decode the HTTP Entity into a value of your desired type (Person). The directive conveniently takes a second argument in the form of a function from Person to whatever you choose to do with it (in our case to complete an HTTP response).
+ - Inside the `complete` directive, you can pass anything that can be marshalled to an HTTP response. Because we have the implicit JSON formats in scope and because we added SprayJsonSupport, the compiler can automatically turn a UserAdded data structure into not only a JSON string, but in a complete HTTP response with the correct status code, etc.
+
+
+At this point, we should be ready to start the server and issue HTTP requests to the server. If you want to send HTTP requests easily, I really like HTTPie (which formats the responses much better), but you can also use cURL as well.
+
+```bash
+echo '{"name":"Daniel", "age": 56}' | http post localhost:8081/api/user
+```
+```http
+HTTP/1.1 200 OK
+Content-Length: 71
+Content-Type: application/json
+Date: Tue, 15 Sep 2020 09:30:02 GMT
+Server: akka-http/10.2.0
+
+{
+ "id": "29ba99ff-4032-433b-be32-3320978a9810",
+ "timestamp": 1600162202286
+}
+```
+
+Alternatively, with cURL:
+```bash
+$ curl \
+ -XPOST \
+ -H "Content-Type: application/json" \
+ -d "{\"name\":\"Daniel\", \"age\":56}" http://localhost:8081/api/user
+
+{"id":"ec72aefa-a68a-45e4-ae30-7cdec81fb959","timestamp":1600162164182}
+```
+
+## 2. Circe
+
+The Circe library is very popular within the Typelevel ecosystem. However, it only works well with the Typelevel libraries, so porting it to Akka HTTP is not straightforward. Thankfully, Heiko Seeberger created this cute repo of JSON libraries for Akka HTTP, just in case Spray-json is not enough for you. You can also dive into the code of the libraries, it's not that long - just a few implicit wrappers over the given functionality of Circe (and other JSON libraries). The JSON libray support for Circe ships in its own distribution, so you'll need to add
+
+```scala
+val akkaHttpJsonSerializersVersion = "1.34.0"
+libraryDependencies ++= Seq(
+
+ // ... the other libraries here
+
+ // add this
+ "de.heikoseeberger" %% "akka-http-circe" % akkaHttpJsonSerializersVersion
+}
+```
+
+At this point, you should be ready to use Circe with Akka HTTP. Here's how you do it:
+
+Step 1. Add the `FailFastCirceSupport` trait as a mix-in to your main application object. This will bring the necessary implicits so that the directives can find the marshallers (serializers) between the HTTP entities that Akka HTTP understands and the internal formats of Circe. Pretty much similar to step 3 of the Spray-json integration.
+
+Step 2. Add an import so that Circe can automatically generate an implicit encoder/decoder pair for the types you want to support (case classes, usually):
+
+```scala
+import io.circe.generic.auto._
+```
+
+And that's it! The rest of the code can stay identical. The code looks like this:
+
+```scala
+object AkkaHttpCirceJson extends FailFastCirceSupport {
+ import io.circe.generic.auto._
+
+ implicit val system = ActorSystem(Behaviors.empty, "AkkaHttpJson")
+
+ val route: Route = (path("api" / "user") & post) {
+ entity(as[Person]) { person =>
+ complete(UserAdded(UUID.randomUUID().toString, System.currentTimeMillis()))
+ }
+ }
+
+ def main(args: Array[String]): Unit = {
+ Http().newServerAt("localhost", 8082).bind(route)
+ }
+}
+```
+
+Same requests will give us the similar responses:
+
+```bash
+echo '{"name":"Daniel", "age": 56}' | http post localhost:8082/api/user
+```
+```http
+HTTP/1.1 200 OK
+Content-Length: 71
+Content-Type: application/json
+Date: Tue, 15 Sep 2020 09:33:20 GMT
+Server: akka-http/10.2.0
+
+{
+ "id": "8de03e1a-913c-4063-80db-8c4b6317715b",
+ "timestamp": 1600162400694
+}
+```
+
+Alternatively, with cURL:
+```bash
+$ curl \
+ -XPOST \
+ -H "Content-Type: application/json" \
+ -d "{\"name\":\"Daniel\", \"age\":56}" http://localhost:8082/api/user
+
+{"id":"ef64fc3b-086c-424f-8404-5c389bf6ca74","timestamp":1600162424186}
+```
+
+
+## 3. Jackson
+
+Yes, the popular library simply known as "JSON for Java" (or the JVM). Fortunately, we're now warmed up and ready to import it from the same repo:
+
+```scala
+libraryDependencies ++= Seq(
+
+ // ... the other libraries here
+
+ // add this
+ "de.heikoseeberger" %% "akka-http-jackson" % akkaHttpJsonSerializersVersion
+}
+```
+
+After including it, you can now use it as well. The usage is even simpler in this case: all you have to do is add the Jackson support trait to your main application object so that the compiler can build the implicits that the Akka HTTP directives will need to marshal/unmarshal entities to/from your types. The support trait is called (unsurprisingly) `JacksonSupport`, so the code will look like this:
+
+```scala
+object AkkaHttpJackson extends JacksonSupport {
+ implicit val system = ActorSystem(Behaviors.empty, "AkkaHttpJson")
+
+ val route: Route = post {
+ entity(as[Person]) { person =>
+ complete(UserAdded(UUID.randomUUID().toString, System.currentTimeMillis()))
+ }
+ }
+
+ def main(args: Array[String]): Unit = {
+ Http().newServerAt("localhost", 8083).bind(route)
+ }
+}
+```
+
+And once you start your server, the POST commands will look the same:
+
+```bash
+echo '{"name":"Daniel", "age": 56}' | http post localhost:8083/api/user
+```
+```http
+HTTP/1.1 200 OK
+Content-Length: 71
+Content-Type: application/json
+Date: Tue, 15 Sep 2020 09:34:35 GMT
+Server: akka-http/10.2.0
+
+{
+ "id": "f1ec08f8-382c-488a-8ba0-195ef2ba3aa2",
+ "timestamp": 1600162475889
+}
+```
+
+Alternatively, with cURL:
+```bash
+$ curl \
+ -XPOST \
+ -H "Content-Type: application/json" \
+ -d "{\"name\":\"Daniel\", \"age\":56}" http://localhost:8083/api/user
+
+{"id":"fbddb191-a28c-446a-9ec5-e91c3d69ed98","timestamp":1600162458223}
+```
+
+## Conclusion
+
+Akka HTTP is a magical library for spinning up HTTP services very quickly. In this article, you learned how to use not 1, but 3 different libraries for serializing and deserializing JSON auto-magically with directives.
+
+Enjoy JSONs!
diff --git a/_posts/2020-09-18-lens.md b/_posts/2020-09-18-lens.md
new file mode 100644
index 000000000000..18470a288b63
--- /dev/null
+++ b/_posts/2020-09-18-lens.md
@@ -0,0 +1,177 @@
+---
+title: "Lenses, Prisms and Optics in Scala"
+date: 2020-09-18
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, monocle]
+excerpt: "A really nice way to inspect, extract and modify deeply nested data structures in Scala."
+---
+This article is for Scala programmers of all levels, who are big fans of the DRY principle . We're going to explore deeply nested data structures using the Monocle library and the concepts of "optics" in Scala.
+
+## Background
+
+Monocle is a popular library for traversing, inspecting and editing deeply nested data structures. In order to use Monocle, add the following to your build.sbt file:
+
+```scala
+libraryDependencies ++= Seq(
+ "com.github.julien-truffaut" %% "monocle-core" % monocleVersion,
+ "com.github.julien-truffaut" %% "monocle-macro" % monocleVersion
+)
+```
+
+## 1. Lenses
+
+Monocle was invented because nested data structures are a pain to inspect and change. The pain increases with the depth of the data structures. Consider the following scenario: we're designing an online web compendium of rock bands (Rock the JVM, right?). We're thinking about the following data structure design:
+
+```scala
+case class Guitar(make: String, model: String)
+case class Guitarist(name: String, favoriteGuitar: Guitar)
+case class RockBand(name: String, yearFormed: Int, leadGuitarist: Guitarist)
+```
+
+Let's assume now that we've created some bands for our database:
+
+```scala
+val metallica = RockBand("Metallica", 1981, Guitarist("Kirk Hammett", Guitar("ESP", "M II")))
+```
+
+Let's also assume that we have a giant database of guitars, and we want to store them in a consistent format. To comply with that format, we'll need to replace all spaces in a guitar's model with a dash (don't ask why). Normally, we'd have to go through the entire data structure and copy everything up to the guitar's model:
+
+```scala
+val metallicaFixed = metallica.copy(
+ leadGuitarist = metallica.leadGuitarist.copy(
+ favoriteGuitar = metallica.leadGuitarist.favoriteGuitar.copy(
+ model = metallica.leadGuitarist.favoriteGuitar.model.replace(" ", "-")
+ )
+ )
+)
+```
+
+This is a pain. Imagine we'd have 10 places in our small app where we would have to do this. The code would be a mess.
+
+This is where Monocle comes in. Monocle gives us the capability to access a deeply nested field in a data structure, inspect it and/or change it, therefore creating a new data structure as a result.
+
+```scala
+val kirksFavGuitar = Guitar("ESP", "M II")
+
+import monocle.Lens
+import monocle.macros.GenLens
+
+val guitarModelLens: Lens[Guitar, String] = GenLens[Guitar](_.model)
+// inspecting
+val kirksGuitarModel = guitarModelLens.get(kirksFavGuitar) // "M II"
+// modifying
+val formattedGuitar = guitarModelLens.modify(_.replace(" ", "-"))(kirksFavGuitar) // Guitar("ESP", "M-II")
+```
+
+So far, this code has the same utility as accessing a field or copying a case class instance. The power of lenses becomes apparent when we compose those lenses:
+
+```scala
+val leadGuitaristLens = GenLens[RockBand](_.leadGuitarist)
+val guitarLens = GenLens[Guitarist](_.favoriteGuitar)
+val guitarModelLens = GenLens[Guitar](_.model)
+val composedLens = leadGuitaristLens.composeLens(guitarLens).composeLens(guitarModelLens)
+```
+
+The resulting Lens now has the capacity to inspect the Metallica band right down to Kirk's favorite guitar model, and change it if we want:
+
+```scala
+val kirksGuitarModel2 = composedLens.get(metallica)
+val metallicaFixed2 = composedLens.modify(_.replace(" ", "-"))(metallica)
+```
+
+Now with the lens in place, we can use it everywhere we need to run similar transformations. We aren't repeating the bulky code for copying case classes.
+
+Why is this pattern called "lens"? Because it allows us to "zoom" into the deeply buried fields of data structures, then inspect or modify them there.
+
+## 2. Prisms
+
+Prisms are another interesting tool for manipulating data structures. This time, we're working in the world of hierarchies, usually sealed classes/traits or enums. Here's a scenario: we're working on a visual design app and we have various built-in shapes in place. We'd like to be able to manipulate their features while still working against the main "interface".
+
+```scala
+sealed trait Shape
+case class Circle(radius: Double) extends Shape
+case class Rectangle(w: Double, h: Double) extends Shape
+case class Triangle(a: Double, b: Double, c: Double) extends Shape
+
+val aCircle = Circle(20)
+val aRectangle = Rectangle(10, 20)
+val aTriangle = Triangle(3,4,5)
+
+val shape: Shape = aCircle
+```
+
+In this scenario, we'd like to be able to increase the radius of this shape if it's a Circle, and leave it intact otherwise - all without having to resort to `isInstanceOf`. Of course, we can do pattern matching:
+
+```scala
+val newCircle: Shape = shape match {
+ case Circle(r) => Circle(r + 10)
+ case x => x
+}
+```
+
+But again, if we wanted to apply this transformation to many Shapes throughout various parts of our code, we'd have no choice but to repeat this pattern. Enter prisms:
+
+```scala
+import monocle.Prism
+val circlePrism = Prism[Shape, Double] {
+ case Circle(r) => Some(r)
+ case _ => None
+}(r => Circle(r))
+```
+
+A Prism takes two argument lists, each of which takes a function. One is of type `Shape => Option[Double]`, so a "getter" (we return an Option because the Shape might be something other than a Circle). The other function is a "creator", of type `Double => Shape`. In other words, a Prism is a wrapper over a back-and-forth transformation between a Double and a Shape. A prism allows us to investigate a Shape and get a double, or use a double and create a Shape.
+
+```scala
+val circle = circlePrism(30) // returns a Shape (actually a Circle)
+val noRadius = circlePrism.getOption(aRectangle) // will return None because that shape is not a Circle
+val radius = circlePrism.getOption(aCircle) // returns Some(20)
+```
+
+This seems complicated at first, but it clears a lot of boilerplate, for several reasons:
+
+ - the prism's apply method acts as a "smart constructor" which can instances of Circle for us
+ - we can safely inspect any shape's radius even if it's not a Circle - this saves us the need to repeat the earlier pattern matching
+
+Both of the above can be used at any point inside our application, without the need to type-check or pattern match every time.
+
+Why is this pattern called a "prism"? Because from the many types (facets) out of a hierarchy of data structures (prism), we're interested in manipulating a single subtype (a "face"). Together with the lens pattern above and with a bunch of others, the Monocle library describes itself as an "optics" library for Scala.
+
+## 3. Composing Optics
+
+Probably the most powerful feature of Monocle is the ability to compose the above patterns (and others). We can inspect and/or modify nested data structures by combining the capability to zoom in (lens) and to isolate a type (prism).
+
+Imagine somebody is designing a brand identity with our visual design app:
+
+```scala
+case class Icon(background: String, shape: Shape)
+case class Logo(color: String)
+case class BrandIdentity(logo: Logo, icon: Icon)
+```
+
+If we want to change tha radius of the icon of a brand - assuming it's a circle, or leave it intact otherwise - we would create the appropriate accessors (lenses) and modifiers for our desired type (prism):
+
+```scala
+val iconLens = GenLens[BrandIdentity](_.icon)
+val shapeLens = GenLens[Icon](_.shape)
+// compose all
+val brandCircleR = iconLens.composeLens(shapeLens).composePrism(circlePrism)
+```
+
+With the above in place, we can take some brands and apply a transformation:
+
+```scala
+val aBrand = BrandIdentity(Logo("red"), Icon("white", Circle(45)))
+val enlargeRadius = brandCircleR.modify(_ + 10)(aBrand)
+// ^^ a new brand whose icon circle's radius is now 55
+
+val aTriangleBrand = BrandIdentity(Logo("yellow"), Icon("black", Triangle(3,4,5)))
+brandCircleR.modify(_ + 10)(aTriangleBrand)
+// ^^ doesn't do anything because the shape isn't a triangle, but the code is 100% safe
+```
+
+All of the data access and manipulation is now reusable throughout the entire application!
+
+## Conclusion
+
+You now know the optics approach to accessing, inspecting and modifying nested data structures. Let me know if you liked it, and I'll write a follow-up to it with more advanced usage, including collections, isomorphisms and integration with Cats!
diff --git a/_posts/2020-09-25-new-types-scala-3.md b/_posts/2020-09-25-new-types-scala-3.md
new file mode 100644
index 000000000000..e6f25e7b4d5e
--- /dev/null
+++ b/_posts/2020-09-25-new-types-scala-3.md
@@ -0,0 +1,175 @@
+---
+title: "New Types in Scala 3"
+date: 2020-09-25
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3, type system]
+excerpt: "Scala 3 introduces some new kinds of types, which we're eagerly awaiting for."
+---
+This article is for Scala programmers of all levels, although some of the later parts and questions will be a tad more difficult. We will start exploring the new features that Scala 3 brings, as well as changes in style, syntax, or deprecations.
+
+The focus of this article is some of the new kinds of types now allowed in Scala 3.
+
+This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## 1. Literal Types
+
+This feature was technically introduced in Scala 2.13, but we will explore it here in the context of the types we're going to talk about next. In short, Scala is now able to treat literal values as singletons of their own types. Remember type-level programming ? We needed to defined each "number" as its own type. Now, Scala can do this by default. The catch is that you'll now have to declare it explicitly:
+
+```scala
+val aNumber = 3 // an Int
+val three: 3 = 3
+```
+
+If you don't specify the type of your val, the compiler will infer the type based on the right-hand side as it did before. If you specify the literal type yourself, the compiler will automatically build a new type behind the scenes, based on what you declared. Any literal can become its own type, and it will be a subtype of its "normal" type, e.g. the type 3 will be a subtype of Int:
+
+```scala
+def passNumber(n: Int) = println(n)
+passNumber(aNumber)
+passNumber(three) // correct, d'oh
+```
+
+However, once you declare that a method/function takes a literal type as argument, nothing but that value will be accepted:
+
+```scala
+def passStrict(n: 3) = println(n)
+passStrict(3)
+// passStrict(aNumber) // not correct
+```
+
+Literal types can be defined for other numerical types (e.g. doubles), for booleans or for strings. Side note: even though String is a reference type, String interning happens automatically for String literals, and Scala can take advantage of it.
+
+```scala
+val myFavoriteLanguage: "Scala" = "Scala"
+val pi: 3.14 = 3.14
+val truth: true = true
+```
+
+Why are literal types useful at all? Literal types are used to enforce compile-time checks to your definitions so that you don't go reckless with your code.
+
+```scala
+def doSomethingWithYourLife(meaning: Option[42]): Unit =
+ meaning.foreach(m => s"I've made it: $m")
+```
+
+The method above will only take Some(42) or None, nothing in between. It's often a good idea to restrict your type declarations to literal types when you're certain that your particular value is critical for your application's logic. In this way, you'll avoid the error of passing an invalid value, and everything will be caught at compile time.
+
+## 2. Union Types
+
+This is a completely new addition in Scala 3. It allows you to use a combined type ("either A or B") in the same declaration. However, this has nothing to do with the `Either` monadic type. Here's an example:
+
+```scala
+def ambivalentMethod(arg: String | Int) = arg match {
+ case _: String => println(s"a String: $arg")
+ case _: Int => println(s"an int: $arg")
+}
+```
+
+This method happily receives either an Int argument or a String argument. This was not possible in the prior versions of Scala. We can use the method as follows:
+
+```scala
+ambivalentMethod(33)
+ambivalentMethod("33")
+```
+
+The caveat of having a union type in a method/function is that in order to use the types properly, e.g. use their methods, we have to pattern match the union-typed value against the possible types.
+
+I mentioned we would discuss literal types in the context of Scala 3, because in conjunction with union types, literal types unlock a nice piece of functionality. Languages like TypeScript have had this for years, and it's really nice that we can have the equivalent in Scala:
+
+```scala
+type ErrorOr[T] = T | "error" // this puts some restraints on what you can do with your values
+def handleResource(file: ErrorOr[File]): Unit = { // we'll discuss braceless syntax in another article
+ // your code here
+}
+```
+
+Inside this method, you can't access the API of the argument "good" type (i.e. File in this case) until you've dealt with the error case. This prevents you from being reckless in your code and forces you to treat unwanted values properly. The best part? Everything happens at compile time.
+
+One more thing about union types. They are not inferred automatically by the compiler if an expression can return multiple types. In other words, the compiler will still compute the lowest common ancestor type by way of inheritance. That said, we can define a union type explicitly if we wanted, and the compiler will be happy too:
+
+```scala
+val stringOrInt = if (43 > 0) "a string" else 43 // Any, as inferred by the compiler
+val aStringOrInt: String | Int = if (43 > 0) "a string" else 43 // OK
+```
+
+## 3. Intersection Types
+
+By way of symmetry, we also have intersection types in Scala 3 now. While you can think of union types as "either A or B", intersection types can be read as "both A and B". Here's an example:
+
+```scala
+trait Camera {
+ def takePhoto(): Unit = println("snap")
+}
+trait Phone {
+ def makeCall(): Unit = println("ring")
+}
+
+def useSmartDevice(sp: Camera & Phone): Unit = {
+ sp.takePhoto()
+ sp.makeCall()
+}
+```
+
+Inside the `useSmartDevice` method, the compiler guarantees that the argument will adhere to both the Camera and the Phone APIs, so you can use both types' methods inside. The intersection type will also act as a type restriction, so we need to mix in both traits if we are to use this method properly:
+
+```scala
+class Smartphone extends Camera with Phone
+
+useSmartDevice(new Smartphone) // cool!
+```
+
+Naturally, an intersection type will be a subtype of both types involved.
+
+An interesting question that might come up is: what if the two types share a method definition? The answer is that the compiler doesn't care. The real type that will be passed to such a method will need to solve the conflict. In other words, the real type that will be used will only have a single implementation of that method, so there will be no conflict at the call site. We're going to address the exact ways Scala solves this problem and the diamond problem with trait linearization in another article.
+
+Another interesting question is: what if the two types share a method signature except the returned types? Assume we have the modules below:
+
+```scala
+trait HostConfig
+trait HostController {
+ def get: Option[HostConfig]
+}
+
+trait PortConfig
+trait PortController {
+ def get: Option[PortConfig]
+}
+```
+
+And assume we want to mix them in our main web server:
+
+```scala
+def getConfigs(controller: HostController & PortController) = controller.get
+```
+
+First question: does this code even compile?
+
+Yes, it does. The `get` method is common between the two types, so we should be able to use it.
+
+Second question: what type does the new `get` method return? What type does this big `getConfigs` method return?
+
+This is a tricky one. Because the argument is of type `HostController & PortController`, any real type that can extend both HostController and PortController must implement the `get` method such that it returns both an `Option[HostConfig]` and `Option[PortConfig]`. The only solution is to make `get` return `Option[HostConfig] & Option[PortConfig]`. The compiler is able to figure this out, and you can be explicit about it:
+
+```scala
+def getConfigs(controller: HostController & PortController)
+ : Option[HostConfig] & Option[PortConfig]
+ = controller.get
+```
+
+Third question: does an intersection type play nice with variance?
+
+Yes, it does. Because Option is covariant, it means that subtyping between Options matches the subtyping between generic types. In other words, `Option[A] & Option[B]` is the same as `Option[A & B]`. We can also change our method as such:
+
+```scala
+def getConfigs(controller: HostController & PortController)
+ : Option[HostConfig & PortConfig]
+ = controller.get
+```
+
+And the code still compiles.
+
+## Conclusion
+
+We explored 2 new types that Scala 3 brings to the table: intersection types and union types. Combined with literal types which were released with Scala 2.13, we are going to see much more powerful and expressive APIs real soon.
+
+Let me know if you liked this article, and I'll write more articles on Scala 3!
diff --git a/_posts/2020-09-29-enums-scala-3.md b/_posts/2020-09-29-enums-scala-3.md
new file mode 100644
index 000000000000..8cff40d09fa7
--- /dev/null
+++ b/_posts/2020-09-29-enums-scala-3.md
@@ -0,0 +1,110 @@
+---
+title: "Enums in Scala 3"
+date: 2020-09-29
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3]
+excerpt: "Scala 3 introduces enums. About time! Even though it might seem like something minor, it has important implications."
+---
+
+This article is for Scala programmers of all levels, and particularly for those Scala programmers who have been emulating enums in Scala 2 for the longest time. Your day has come, because Scala 3 now supports enums out of the box. This will be the focus of this article.
+
+This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## Background
+
+Scala 2 famously had no support for enums. We had to bend over backwards to support it:
+
+```scala
+object Permissions extends Enumeration {
+ val READ, WRITE, EXEC, NONE = Value
+}
+```
+
+What on Earth was that? Value?!
+
+We won't even get into the weeds there, it makes no sense. This construct was probably one of the weirdest parts of Scala 2.
+
+## Enter Enums
+
+Finally, Scala has first-class enums like any standard programming language:
+
+```scala
+enum Permissions {
+ case READ, WRITE, EXEC
+}
+```
+
+There you go. There's one `case` which might or might not have been needed, but we can't be too picky. Enums are now first-class, and we can use them as in Java or other languages:
+
+```scala
+val read: Permissions = Permissions.READ
+```
+
+Under the hood, the compiler generates a sealed class and 3 well-defined values in its companion object.
+
+## Enums with Arguments
+
+As was expected, enums can also have arguments, and the constants will have to be declared with a given expression:
+
+```scala
+enum PermissionsWithBits(bits: Int) {
+ case READ extends PermissionsWithBits(4) // binary 100
+ case WRITE extends PermissionsWithBits(2) // binary 010
+ case EXEC extends PermissionsWithBits(1) // binary 001
+ case NONE extends PermissionsWithBits(0)
+}
+```
+
+The construct is again a bit boilerplate-y - especially since we can't extend anything except the wrapping enum - but we can't complain. In the above example we have an enum with 4 possible values. Of course, we can access their field with the regular accessor syntax.
+
+## Fields and Methods
+
+Enums can contain fields and methods, just like a normal class - they're compiled to a sealed class after all. We can define them inside the enum body and we can access them with the regular dot-accessor syntax.
+
+```scala
+enum PermissionsWithBits(bits: Int) {
+ // the cases here
+
+ def toHex: String = Integer.toHexString(bits) // the java way of impl
+ // can also define other members, e.g. vals
+}
+```
+
+One interesting thing is that we can also define variables (vars) inside the enum. This might come in conflict with the immovable aspect of enums. I would certainly not recommend creating variables inside enums - it would be like defining global variables, free for anyone in any point of the code to change.
+
+A nice addition to enums is the ability to create companion objects, where we can define "static" fields and methods, perhaps "smart" constructors:
+
+```scala
+object PermissionsWithBits {
+ def fromBits(bits: Int): PermissionsWithBits = // do your bit checking
+ PermissionsWithBits.NONE
+}
+```
+
+## Standard API
+
+Enums come with some predefined utility methods. First, the ability to check the "index" of a given enum value inside the "order" of definition of cases:
+
+```scala
+// if you want to convert to an integer (the order of the enum instance)
+val indexOfRead = Permissions.READ.ordinal
+```
+
+Second, the ability to fetch all possible values of an enum type, perhaps to iterate over them or to consider all at once:
+
+```scala
+val allPermissions = Permissions.values
+```
+
+Third, the ability to convert a String into an enum value:
+
+```scala
+val readPermission = Permissions.valueOf("READ")
+```
+
+## Conclusion
+
+And with that, you should be set! Scala 3 is now in line with many other languages in its capability to define enums. You should now be able to safely define and use enums, add parameters, methods and fields and use their pre-defined APIs.
+
+Let me know if you liked this article, and I'll write more articles on Scala 3!
diff --git a/_posts/2020-09-29-scala-3-traits.md b/_posts/2020-09-29-scala-3-traits.md
new file mode 100644
index 000000000000..9c1981161e6c
--- /dev/null
+++ b/_posts/2020-09-29-scala-3-traits.md
@@ -0,0 +1,118 @@
+---
+title: "Scala 3 Traits: New Features"
+date: 2020-09-29
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3, traits]
+excerpt: "This article will continue some of the previous explorations of Scala 3. Here, we'll discuss some of the new functionality of traits in Scala 3."
+---
+
+This article will continue some of the previous explorations of Scala 3. Here, we'll discuss some of the new functionality of traits in Scala 3.
+
+This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## 1. Background
+
+Scala traits were originally conceived to be analogous to Java interfaces. Essentially, a trait was a type definition which wrapped a suite of abstract fields and methods. In time, traits acquired additional functionality and features, such as non-abstract fields and methods. This led to some legitimate questions around the [boundary between abstract classes and traits](https://www.youtube.com/watch?v=_7ULjOILxhI).
+
+That line will get even blurrier with the arrival of Scala 3.
+
+## 2. Trait Arguments
+
+One of the practical differences between abstract classes and traits was (in Scala 2) that traits could not receive constructor arguments. Put simply, now they can:
+
+```scala3
+trait Talker(subject: String) {
+ def talkWith(another: Talker): String
+}
+```
+
+Extending such a trait looks just like extending a regular class:
+
+```scala3
+class Person(name: String) extends Talker("rock")
+```
+
+Enhancing traits with parameters certainly has its advantages. However, this may pose some problems. The first problem is that sometimes in large code bases (and not only), extending the same trait multiple times is not unheard of. What happens if you mix-in a trait with one argument in one place, and with another argument in another place?
+
+The short answer is that won't compile. The rule is: if a superclass already passes an argument to the trait, if we mix it again, we must not pass any argument to that trait again.
+
+```scala3
+class RockFan extends Talker("rock")
+class RockFanatic extends RockFan with Talker // must not pass argument here
+```
+
+Another problem is: what happens if we define a trait hierarchy? How should we pass arguments to derived traits?
+
+Again, short answer: derived traits will not pass arguments to parent traits:
+
+```scala3
+trait BrokenRecord extends Talker
+```
+
+That's a rule. Passing arguments to parent traits will not compile.
+
+Cool, but how are we now supposed to mix this trait into one of our classes? Say we wanted to create a class which denotes this person we all have in our family or our circle of friends, who talks until they turn pale.
+
+```scala3
+class AnnoyingFriend extends BrokenRecord("politics")
+```
+
+This is illegal, because the BrokenRecord trait doesn't take arguments. But then how are we supposed to pass the right argument to the Talker trait?
+
+The answer is by mixing it again:
+
+```scala3
+class AnnoyingFriend extends BrokenRecord with Talker("politics")
+```
+
+A bit clunky, but that's the only way to make the type system sound with respect to this new capability of traits.
+
+## 3. Transparent Traits
+
+The Scala compiler's type inference is one of its most powerful features. However, without enough information, sometimes even the compiler's type inference isn't powerful enough. Here's an example:
+
+```scala3
+trait Color
+case object Red extends Color
+case object Green extends Color
+case object Blue extends Color
+
+val color = if (43 > 2) Red else Blue
+```
+
+Can you guess what the inferred type of `color` is? Spoiler: it's not `Color`.
+
+Which is weird, right? We'd expect the inferred type to be the lowest common ancestor of the two types, Red and Blue. The complete inferred type is `Color with Product with Serializable`. The reason is that both Red and Blue derive from Color, but because they are `case object`s, they automatically implement the traits `Product` (from Scala) and `Serializable` (from Java). So the lowest common ancestor is the combination of all three.
+
+The thing is that we rarely use the traits `Product` or `Serializable` as standalone types we attach to values. So Scala 3 allows us to ignore these kinds of traits in type inference, by making them a `transparent` trait. Here's an example. Assume we have the following definitions for a graphical library:
+
+```scala3
+trait Paintable
+trait Color
+object Red extends Color with Paintable
+object Green extends Color with Paintable
+object Blue extends Color with Paintable
+```
+
+(Notice we did not make them `case object`s for brevity. We'll come back to it.)
+
+Assume further that the trait `Paintable` is rarely used as a standalone trait, but rather as an auxiliary trait in our library definitions. In this case, if we were to say
+
+```scala3
+val color = if (43 > 2) Red else Blue
+```
+
+then we'd like the type inference to detect `color` as being of type `Color`, not `Color with Paintable`. We can suppress `Paintable` from type inference by marking it with `super`:
+
+```scala3
+transparent trait Paintable
+```
+
+After that we'll see that our variable `color` is now marked as `Color`.
+
+When Scala 3 comes out, the traits `Product`, `Comparable` (from Java) and `Serializable` (from Java) will be automatically be treated as transparent traits in the Scala compiler. Of course, if you mark your value as having a particular type, transparent traits will not influence the type checker.
+
+## 4. Conclusion
+
+You've learned two new features of Scala 3 regarding traits. Put them to good use when Scala 3 comes out!
diff --git a/_posts/2020-10-02-scala-3-opaque.md b/_posts/2020-10-02-scala-3-opaque.md
new file mode 100644
index 000000000000..259ae772cf83
--- /dev/null
+++ b/_posts/2020-10-02-scala-3-opaque.md
@@ -0,0 +1,107 @@
+---
+title: "Scala 3: Opaque Types"
+date: 2020-10-02
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3]
+excerpt: "We discover opaque type aliases in Scala 3 and how we can define new types with zero overhead."
+---
+
+This article continues the series on Scala 3. I'll assume you're familiar with some of the Scala (version 2) foundations, such as defining basic classes, methods, type aliases.
+
+This article focuses on a small but exciting feature: opaque types. This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## 1. Background and Motivation
+
+It is often the case that we define new types as wrappers over existing types (composition). However, in many cases this mechanism will involve at least some sort of overhead, while accessing fields, methods or composing these new types.
+
+Let's take an example. Say we're working on a social network. One of the fundamental pieces of data is a user's details, but we want to enforce some rules so that the user's details are correct. For instance, we may want to force their names to start with a capital letter (this may not be true in all languages, but let's take this scenario for the sake of the example).
+
+```scala3
+case class Name(value: String) {
+ // some logic here
+}
+```
+
+This case is a plain wrapper over a String. Of course, we can also use things like [refined types](https://www.youtube.com/watch?v=IDrGbsupaok), but regardless of what we end up choosing, this new Name type incurs some sort of overhead. If we have millions of users, these tiny overheads will start to add up.
+
+## 2. Enter Opaque Types
+
+A name is really just a string, but due to the extra logic we attach to it, we have no choice but to incur this overhead, either by wrapping the string, or by forcing the compiler to run some extra checks at compile time. Opaque types allow us to define Name as *being* a String, and allows us to also attach functionality to it:
+
+```scala3
+object SocialNetwork {
+ opaque type Name = String
+}
+```
+
+We've defined a type alias, which we can now freely use interchangeably with String inside the scope it's defined.
+
+## 3. Defining an Opaque Type's API
+
+The benefit of an opaque type is that we can treat this new type like a standalone type, such as a class or trait. That means we can define a companion object.
+
+```scala3
+object SocialNetwork {
+ opaque type Name = String
+
+ object Name {
+ def fromString(s: String): Option[Name] =
+ if (s.isEmpty || s.charAt(0).isLower) None else Some(s) // simplified
+ }
+}
+```
+
+The idea with an opaque type is that you can only interchange it with String in the scope it's defined, but otherwise the outside world has no idea that a Name is in fact a String. To the outside scope, Name is a completely different type with its own API (currently none). This allows you to start with a new type being implemented in terms of an existing type (String) with zero boilerplate or overhead. On the other hand, the new type is treated as having no connection to the type it's implemented as. The line below will not compile:
+
+```scala3
+val name: Name = "Daniel" // expected Name, got String
+```
+
+In this way, we have some good news and bad news. The bad news is that this new type has no API of its own. Even if it's implemented as a String, you don't have access to any String methods. However, the good news is that you now have a fresh zero-overhead type whose API you can write from scratch.
+
+The API for the new type will have to be defined as extension methods. We'll talk extension methods in another article, but the structure will look like this:
+
+```scala3
+// still within the SocialNetwork scope where Name is defined
+extension (n: Name) {
+ def length: Int = n.length
+}
+```
+
+Having defined some basic "static" API (i.e. companion object) and "non-static" API (i.e. extension methods), we're now ready to use our new type:
+
+```scala3
+val name: Option[Name] = Name.fromString("Daniel")
+val nameLength = name.map(_.length)
+```
+
+## 4. Opaque Types with Bounds
+
+Opaque type definitions can have type restrictions, much like regular type aliases. Let's imagine we're working on a graphics library and we deal with colors:
+
+```scala3
+object Graphics {
+ opaque type Color = Int // in hex
+ opaque type ColorFilter <: Color = Int
+
+ val Red: Color = 0xff000000
+ val Green: Color = 0xff0000
+ val Blue: Color = 0xff00
+ val halfTransparency: ColorFilter = 0x88
+}
+```
+
+We can then use Color and ColorFilter in the same style as we use a class hierarchy, in what concerns the possible substitutions:
+
+```scala3
+import Graphics._
+case class Overlay(c: Color)
+
+// ok, because ColorFilter "extends" Color
+val fadeLayer = Overlay(halfTransparency)
+```
+
+## 5. Conclusion
+
+You've learned a new tool in the Scala 3 arsenal. It has its drawbacks - notably the inability to use the underlying type's API outside the alias definition scope - but it allows much more flexibility in what you can express in terms of existing types with zero overhead.
diff --git a/_posts/2020-10-05-scala-types-kinds.md b/_posts/2020-10-05-scala-types-kinds.md
new file mode 100644
index 000000000000..56641e0dfab5
--- /dev/null
+++ b/_posts/2020-10-05-scala-types-kinds.md
@@ -0,0 +1,122 @@
+---
+title: "Types, Kinds and Type Constructors in Scala"
+date: 2020-10-06
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "Scala has a powerful type system. We look at how Scala types can be organized, what type constructors are, and why we should care."
+---
+
+This article is for the mature Scala developer. We aren't looking at the basics here, and some abstract thinking will be required for this one.
+
+## 1. Background
+
+What's up with these types in Scala? The truth is Scala's type system is extremely powerful, enough to confuse even some of the advanced devs.
+
+Generics in particular are one hard topic, for which we've already posted [some content](https://www.youtube.com/watch?v=b1ftkK1zhxI) (and more to come). The presence of generics in Scala has some important implications in how we think about types. Now, because one of the main strengths of Scala is the increased productivity of devs by leveraging the type system, thinking about types correctly will directly influence how we design our code.
+
+## 2. Level 0: Value Types
+
+Types in Scala can be organized into _kinds_. I'm going to start the abstract path by looking at something really simple: normal types that we can attach to values.
+
+```scala3
+val aNumber: Int = 42
+val aString: String = "Scala"
+```
+
+Types like `Int`, `String`, or regular types that we define (e.g. case classes) can be attached to values. I'll call them level-0 types. This is the simplest _kind_ of types imaginable — we use these types every day in our Scala code.
+
+## 3. Level 1: "Generics"
+
+As our code gets more complicated, we are increasingly interested in reusing our code for many types at once. For example, the logic of a singly linked list is identical, regardless of the type of elements it contains. As such, we attach type arguments to the new type we deeclare:
+
+```scala3
+class LinkedList[T]
+class Optional[T]
+```
+
+We generally call the types such as `LinkedList` or `Optional` above simply "generic". They can work on `Int`s, `String`s, `Person`s and other level-0 types. However, these new types cannot be attached to a value on their own. They need to have a real type argument before we can use a value with them:
+
+```scala3
+val aListOfNumbers: LinkedList[Int] = new LinkedList[Int]
+val aListOfStrings: LinkedList[String] = new LinkedList[String]
+```
+
+So notice we need to use a level-0 type as a type argument before we can use these new types. The type `LinkedList[Int]` is a value type (level-0), because it can be attached to a value. Because we can only use LinkedList after we pass a level-0 type as argument to it, LinkedList is a higher-level type. I'll call it a level-1 type, because it takes type arguments of the inferior kind.
+
+Level-1 is the _kind_ of types which receive type arguments of the inferior level (level-0).
+
+## 4. Type Constructors
+
+Look at how we attached the type `LinkedList[Int]` to the previous value. We used the level-1 type `LinkedList` and we used the level-0 type argument `Int` to create a new level-0 type. Does that sound similar to something else you've seen?
+
+If you think about it, this mechanism looks similar to a function: take a function, pass a value to it, obtain another value. Except in this case, we work with types: take a level-1 type, pass a level-0 type argument to it, obtain another level-0 type.
+
+For this reason, these generic types are also called type constructors, because they can _create_ level-0 types. `LinkedList` itself is a type constructor: takes a value type (e.g. `Int`) and returns a value type (e.g. `LinkedList[Int]`).
+
+## 5. Level 2 and Beyond: Higher-Kinded Types
+
+Up to this point, Scala has similar capabilities to Java. However, the Scala type system moves a step further, by allowing the definitions of generic types _whose type arguments are also generic_. We call these higher-kinded types. I'll call them level-2 types, for reasons I'm going to detail shortly. These type definitions have special syntax:
+
+```scala3
+// the Cats library uses this A LOT
+class Functor[F[_]]
+```
+
+The underscore marks the fact that the type argument F is itself generic (level-1). Because this new type takes a level-1 type argument, the `Functor` example above is a level-2 type. In order to use this type and attach it to a value, we need to use a real level-1 type:
+
+```scala3
+val functorList = new Functor[List]
+val functorOption = new Functor[Option]
+```
+
+Notice we did not pass `List[Int]` as a type argument (which would have been a level-0 type), but rather `List` itself (a level-1 type).
+
+Much like `LinkedList`, `Functor` itself is a type constructor. It can create a value type by passing a level-1 type to it. You can think of `Functor` as similar to a function taking a level-1 type and returning a level-0 type.
+
+Scala is permissive enough to allow even higher-kinded types (in my terminology, level-3 and above) with nested `[_]` structures:
+
+```scala3
+class Meta[F[_[_]]] // a level 3 type
+```
+
+And they would work in a similar fashion - pass a type of an inferior-kind (this case, level 2) to use it:
+
+```scala3
+val metaFunctor = new Meta[Functor]
+```
+
+This example is a bit contrived, because we almost never need to use types beyond level-2. Level-2 types already pretty abstract as they are — although we do try to [smoothen the learning curve](https://rockthejvm.com/p/cats).
+
+## 6. More Type Constructors
+
+Now that you know what a type constructor is, we can expand the concept to types which take multiple type arguments, and perhaps of different _kinds_. Examples below:
+
+```scala3
+class HashMap[K, V]
+val anAddressBook = new HashMap[String, String]
+
+class ComposedFunctor[F[_], G[_]]
+val aComposedFunctor = new ComposedFunctor[List, Option]
+
+class Formatter[F[_], T]
+val aFormatter = new Formatter[List, String]
+```
+
+Given what you've learned so far, you can read these types for what they are:
+
+ - `HashMap` is (by itself) a type constructor taking two level-0 type arguments.
+ - `ComposedFunctor` is (by itself) a type constructor taking two level-1 type arguments.
+ - `Formatter` is (by itself) a type constructor taking a level-1 type argument and a level-0 type argument.
+
+## 7. Conclusion: Why Should We Care?
+
+We've explored how types in Scala are organized and what type constructors are. Why is this important? How does it help us in real life?
+
+Here's the deal.
+
+ 1. When you get started with Scala, you work with normal types, like `Int`s or classes you define, usually plain data structures.
+ 2. As you work with increasingly complex code, you start noticing patterns in your code. Ideally, you'll also have more power (and responsibility) to shape the future direction of your codebase. You'll use higher-level types and generics to accomplish that. It's almost impossible not to.
+ 3. As you become more experienced and notice additional subtle common functionality in your code, you may look to some pure FP libraries like [Cats](https://typelevel.org/cats) to manage logic.
+
+Without good understanding of Scala's type system, not only will this progression seem hard, but you'll also increasingly resist it. As you resist it, you place obstacles to your own growth as a Scala engineer. Conversely, with good understanding of types in Scala, this progression will not only be natural to you, but you'll enjoy your development and abstract code will seem like child's play.
diff --git a/_posts/2020-10-07-scala-3-type-lambdas.md b/_posts/2020-10-07-scala-3-type-lambdas.md
new file mode 100644
index 000000000000..54cba8b1d301
--- /dev/null
+++ b/_posts/2020-10-07-scala-3-type-lambdas.md
@@ -0,0 +1,85 @@
+---
+title: "Type Lambdas in Scala 3"
+date: 2020-10-06
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3, type system]
+excerpt: "Exploring a quick but powerful structure in Scala 3 - type lambdas. This will help you think higher-level."
+---
+
+This article is a bit more difficult — it's aimed at the experienced Scala developer who can think at a higher level. Ideally, you've read the [previous article](https://rockthejvm.com/scala-types-kinds) - it serves as a prep for this one. Type lambdas are simple to express in Scala 3, but the ramifications are deep.
+
+This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## 1. Background
+
+We discussed in the [previous article](/scala-types-kinds) about categorizing types in Scala into _kinds_. Scala 3 is no different here. However, it introduces a new concept and a syntactic structure to express it, which might look daunting and hard to wrap your head around.
+
+To quickly recap:
+
+ - Scala types belong to _kinds_. Think of kinds as _types of types_.
+ - Plain types like `Int`, `String` or your own non-generic classes belong to the _value-level_ kind — the ones you can attach to values.
+ - Generic types like `List` belong to what I called the level-1 kind — they take plain (level-0) types as type arguments.
+ - Scala allows us to express higher-kinded types — generic types whose type arguments are also generic. I called this kind the level-2 kind.
+ - Generic types can't be attached to values on their own; they need the right type arguments (of inferior kinds) in place. For this reason, they're called type constructors.
+
+## 2. Types Look Like Functions
+
+As I mentioned before, generic types need the appropriate type arguments before they can be attached to a value. We can never use the `List` type directly to a value, but only `List[Int]` (or some other concrete type).
+
+You can therefore think of `List` (the generic type itself) as similar to a function, which takes a level-0 type and returns a level-0 type. This "function" from level-0 types to level-0 types represents the _kind_ which `List` belongs to. In Scala 2, representing this such a type was horrible (`{ type T[A] = List[A] })#T`, yuck). In Scala 3, it looks much more similar to a function:
+
+```scala3
+[X] =>> List[X]
+```
+
+Read this structure as "a type that takes a type argument `X` and results in the type `List[X]`". This does the exact same thing as the `List` type (by itself): takes a type argument and results in a new type.
+
+Some more examples in increasing order of complexity:
+
+ - `[T] =>> Map[String, T]` is a type which takes a single type argument `T` and "returns" a `Map` type with `String` as key and `T` as values
+ - `[T, E] =>> Either[Option[T], E]` is a type which takes two type arguments and gives you back a concrete `Either` type with `Option[T]` and `E`
+ - `[F[_]] =>> F[Int]` is a type which takes a type argument which is itself generic (like `List`) and gives you back that type, typed with `Int` (too many types, I know)
+
+## 3. Why We Need Type Lambdas
+
+Type lambdas become important as we start to work with higher-kinded types. Consider Monad, one of the most popular higher-kinded type classes. In its simplest form, it looks like this:
+
+```scala
+trait Monad[M[_]] {
+ def pure[A](a: A): M[A]
+ def flatMap[A, B](m: M[A])(f: A => M[B]): M[B]
+}
+```
+
+You might also know that `Either` is a monadic data structure (another article on that, perhaps), so we can write a `Monad` for it. However, `Either` takes two type arguments, whereas `Monad` requires that its type argument take only one. How do we write it? We would like to write something along the lines of
+
+```scala
+class EitherMonad[T] extends Monad[Either[T, ?]] {
+ // ... implementation
+}
+```
+
+In this way, this `EitherMonad` could work for both `Either[Exception, Int]` and `Either[String, Int]`, for example (where `Int` is the desired type). Given an error type `E`, we'd like `EitherMonad` to work with `Either[E, Int]` whatever concrete `E` we might end up using.
+
+Sadly, the above structure is not valid Scala.
+
+The answer is that we would write something like
+
+```scala 3
+class EitherMonad[T] extends Monad[[E] =>> Either[T, E]] {
+ // ... implementation
+}
+```
+
+It's as if we had a two-argument function, and we needed to pass a partial application of it to another function.
+
+If this is really abstract and hard to wrap your head around, I feel ya.
+
+Prior to Scala 3, libraries like Cats used to resort to compiler plugins ([kind-projector](https://github.com/typelevel/kind-projector)) to achieve something akin to the `?` structure above. Now in Scala 3, it's expressly permitted in the language.
+
+## 4. Conclusion
+
+With a simple syntactic structure, Scala 3 solved a problem that API designers had been facing (and bending over backwards) for ages - how to define higher-kinded types where some type arguments are left "blank".
+
+In a future article, we'll talk about some more advanced capabilities (and pitfalls) of type lambdas.
diff --git a/_posts/2020-10-12-spark-broadcast-joins.md b/_posts/2020-10-12-spark-broadcast-joins.md
new file mode 100644
index 000000000000..483d4b9346b0
--- /dev/null
+++ b/_posts/2020-10-12-spark-broadcast-joins.md
@@ -0,0 +1,190 @@
+---
+title: "Broadcast Joins in Apache Spark: an Optimization Technique"
+date: 2020-10-12
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [spark, optimization]
+excerpt: "Broadcast joins in Apache Spark are one of the most bang-for-the-buck techniques for optimizing speed and avoiding memory issues. Let's take a look."
+---
+
+This article is for the Spark programmers who know some fundamentals: how data is split, how Spark generally works as a computing engine, plus some essential DataFrame APIs.
+
+## 1. Essentials
+
+For this article, we use Spark 3.0.1, which you can either [download](https://spark.apache.org/downloads.html) as a standalone installation on your computer, or you can import as a library definition in your Scala project, in which case you'll have to add the following lines to your build.sbt:
+
+```scala
+val sparkVersion = "3.0.1"
+
+libraryDependencies ++= Seq(
+ "org.apache.spark" %% "spark-core" % sparkVersion,
+ "org.apache.spark" %% "spark-sql" % sparkVersion,
+)
+```
+
+If you chose the standalone version, go ahead and start a Spark shell, as we will run some computations there. If you chose the library version, create a new Scala application and add the following tiny starter code:
+
+```scala
+import org.apache.spark.sql.SparkSession
+
+val spark = SparkSession.builder()
+ .appName("Broadcast Joins")
+ .master("local")
+ .getOrCreate()
+
+val sc = spark.sparkContext
+```
+
+For this article, we'll be using the DataFrame API, although a very similar effect can be seen with the low-level RDD API.
+
+## 2. A Tale of an Innocent Join
+
+Here's the scenario. Let's say we have a huge dataset - in practice, in the order of magnitude of billions of records or more, but here just in the order of a million rows so that we might live to see the result of our computations locally.
+
+```scala
+// large table
+val table = spark.range(1, 100000000) // column is "id"
+```
+
+At the same time, we have a small dataset which can easily fit in memory. For some reason, we need to join these two datasets. Examples from real life include:
+
+ - tagging each row with one of n possible tags, where n is small enough for most 3-year-olds to count to
+ - finding the occurrences of some preferred values (so some sort of filter)
+ - doing a variety of lookups with the small dataset acting as a lookup table
+
+Regardless, we join these two datasets. Let's take a combined example and let's consider a dataset that gives medals in a competition:
+
+```scala
+val rows = sc.parallelize(List(
+ Row(1, "gold"),
+ Row(2, "silver"),
+ Row(3, "bronze")
+))
+
+val rowsSchema = StructType(Array(
+ StructField("id", IntegerType),
+ StructField("medal", StringType)
+))
+
+// small table
+val lookupTable: DataFrame = spark.createDataFrame(rows, rowsSchema)
+```
+
+Having these two DataFrames in place, we should have everything we need to run the join between them. It's easy, and it should be quick, since the small DataFrame is really small:
+
+```scala
+val joined = table.join(lookupTable, "id")
+joined.show()
+```
+
+Brilliant - all is well. Except it takes a bloody ice age to run.
+
+## 3. The Large-Small Join Problem
+
+Why does the above join take so long to run?
+
+If you ever want to debug performance problems with your Spark jobs, you'll need to know how to [read query plans](https://blog.rockthejvm.com/reading-query-plans/), and that's what we are going to do here as well. Let's have a look at this job's query plan so that we can see the operations Spark will perform as it's computing our innocent join:
+
+```scala
+joined.explain()
+```
+
+This will give you a piece of text that looks very cryptic, but it's information-dense:
+
+```perl
+== Physical Plan ==
+*(5) Project [id#259L, medal#264]
++- *(5) SortMergeJoin [id#259L], [cast(id#263 as bigint)], Inner
+ :- *(2) Sort [id#259L ASC NULLS FIRST], false, 0
+ : +- Exchange hashpartitioning(id#259L, 200)
+ : +- *(1) Range (1, 10000000, step=1, splits=6)
+ +- *(4) Sort [cast(id#263 as bigint) ASC NULLS FIRST], false, 0
+ +- Exchange hashpartitioning(cast(id#263 as bigint), 200)
+ +- *(3) Filter isnotnull(id#263)
+ +- Scan ExistingRDD[id#263,order#264]
+```
+
+In this query plan, we read the operations in dependency order from top to bottom, or in computation order from bottom to top. Let's read it top-down:
+
+ - the final computation of the `id` and the `medal` obtained after the join of the two DataFrames, which requires
+ - a sort-merge join on the columns `id` and `id` (with different identifiers under the hash tag), which requires
+ - a sort of the big DataFrame, which comes after
+ - **a shuffle of the big DataFrame**
+ - and a sort + shuffle + small filter on the small DataFrame
+
+The shuffle on the big DataFrame - the one at the middle of the query plan - is required, because a join requires matching keys to stay on the same Spark executor, so Spark needs to redistribute the records by hashing the join column. This is a shuffle. But as you may already know, a shuffle is a massively expensive operation. On billions of rows it can take hours, and on more records, it'll take... more.
+
+## 4. Enter Broadcast Joins
+
+Fundamentally, Spark needs to somehow guarantee the correctness of a join. Normally, Spark will redistribute the records on both DataFrames by hashing the joined column, so that the same hash implies matching keys, which implies matching rows.
+
+There is another way to guarantee the correctness of a join in this situation (large-small joins) by simply duplicating the small dataset on all the executors. In this way, each executor has all the information required to perform the join at its location, without needing to redistribute the data. This is called a broadcast.
+
+```scala
+val joinedSmart = table.join(broadcast(lookupTable), "id")
+joinedSmart.show()
+```
+
+Much to our surprise (or not), this join is pretty much instant. The query plan explains it all:
+
+```perl
+== Physical Plan ==
+*(2) Project [id#294L, order#299]
++- *(2) BroadcastHashJoin [id#294L], [cast(id#298 as bigint)], Inner, BuildRight
+ :- *(2) Range (1, 100000000, step=1, splits=6)
+ +- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)))
+ +- *(1) Filter isnotnull(id#298)
+ +- Scan ExistingRDD[id#298,order#299]
+```
+
+It looks different this time. No more shuffles on the big DataFrame, but a BroadcastExchange on the small one. Because the small one is tiny, the cost of duplicating it across all executors is negligible.
+
+## 5. Automatic Detection
+
+In many cases, Spark can automatically detect whether to use a broadcast join or not, depending on the size of the data. If Spark can detect that one of the joined DataFrames is small (10 MB by default), Spark will automatically broadcast it for us. The code below:
+
+```scala
+val bigTable = spark.range(1, 100000000)
+val smallTable = spark.range(1, 10000) // size estimated by Spark - auto-broadcast
+val joinedNumbers = smallTable.join(bigTable, "id")
+```
+
+produces the following query plan:
+
+```perl
+== Physical Plan ==
+*(2) Project [id#14L]
++- *(2) BroadcastHashJoin [id#14L], [id#12L], Inner, BuildLeft
+ :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false])), [id=#88]
+ : +- *(1) Range (1, 10000, step=1, splits=1)
+ +- *(2) Range (1, 100000000, step=1, splits=1)
+```
+
+which looks very similar to what we had before with our manual broadcast.
+
+However, in the previous case, Spark did not detect that the small table could be broadcast. How come? The reason is that Spark will not determine the size of a local collection because it might be big, and evaluating its size may be an O(N) operation, which can defeat the purpose before any computation is made.
+
+Spark will perform auto-detection when
+
+ - it constructs a DataFrame from scratch, e.g. `spark.range`
+ - it reads from files with schema and/or size information, e.g. Parquet
+
+## 6. Configuring Broadcast Join Detection
+
+The threshold for automatic broadcast join detection can be tuned or disabled. The configuration is `spark.sql.autoBroadcastJoinThreshold`, and the value is taken in bytes. If you want to configure it to another number, we can set it in the SparkSession:
+
+```scala
+spark.conf.set("spark.sql.autoBroadcastJoinThreshold", 104857600)
+```
+
+or deactivate it altogether by setting the value to -1.
+
+```scala
+spark.conf.set("spark.sql.autoBroadcastJoinThreshold", -1)
+```
+
+This is also a good tip to use while testing your joins in the absence of this automatic optimization. We also use this in our [Spark Optimization course](https://rockthejvm.com/p/spark-optimization) when we want to test other optimization techniques.
+
+## 7. Conclusion
+
+Broadcast joins are one of the first lines of defense when your joins take a long time and you have an intuition that the table sizes might be disproportionate. It's one of the cheapest and most impactful performance optimization techniques you can use. Broadcast joins may also have other benefits (e.g. mitigating OOMs), but that'll be the purpose of another article.
diff --git a/_posts/2020-10-14-spark-joins.md b/_posts/2020-10-14-spark-joins.md
new file mode 100644
index 000000000000..112a2b3700fa
--- /dev/null
+++ b/_posts/2020-10-14-spark-joins.md
@@ -0,0 +1,274 @@
+---
+title: "ALL the Joins in Spark DataFrames"
+date: 2020-10-12
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [spark]
+excerpt: "It's obvious that Spark allows us to join tables. What's not obvious is how many different kinds of joins Spark supports. We explore them in this article."
+---
+
+This article is for the beginner Spark programmer. If you're just starting out and you're curious about the kinds of operations Spark supports, this blog post is for you.
+
+## Setup
+
+We use Spark 3.0.1, which you can [download](https://spark.apache.org/downloads.html) to your computer or set up manually as a library in a Scala & SBT project, with the following added to your build.sbt:
+
+```scala
+val sparkVersion = "3.0.1"
+
+libraryDependencies ++= Seq(
+ "org.apache.spark" %% "spark-core" % sparkVersion,
+ "org.apache.spark" %% "spark-sql" % sparkVersion,
+)
+```
+
+If you use the standalone installation, you'll need to start a Spark shell. If you're in a dedicated Scala application, add the following small boilerplate at the start of your code:
+
+```scala
+import org.apache.spark.sql.SparkSession
+
+val spark = SparkSession.builder()
+ .appName("ALL THE JOINS")
+ .master("local")
+ .getOrCreate()
+
+val sc = spark.sparkContext // for creating RDD through parallelize
+```
+
+This article explores the different kinds of joins supported by Spark. We'll use the DataFrame API, but the same concepts are applicable to RDDs as well.
+
+## Joining DataFrames
+
+Let's say we're working on a dataset of kids, where they need to organize into teams to complete a project for school. Assume lots of records in practice, but we'll be working on smaller data here to prove a point. You can copy the following data:
+
+```scala
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+
+val kids = sc.parallelize(List(
+ Row(40, "Mary", 1),
+ Row(41, "Jane", 3),
+ Row(42, "David", 3),
+ Row(43, "Angela", 2),
+ Row(44, "Charlie", 1),
+ Row(45, "Jimmy", 2),
+ Row(46, "Lonely", 7)
+))
+
+val kidsSchema = StructType(Array(
+ StructField("Id", IntegerType),
+ StructField("Name", StringType),
+ StructField("Team", IntegerType),
+))
+
+val kidsDF = spark.createDataFrame(kids, kidsSchema)
+
+val teams = sc.parallelize(List(
+ Row(1, "The Invincibles"),
+ Row(2, "Dog Lovers"),
+ Row(3, "Rockstars"),
+ Row(4, "The Non-Existent Team")
+))
+
+val teamsSchema = StructType(Array(
+ StructField("TeamId", IntegerType),
+ StructField("TeamName", StringType)
+))
+
+val teamsDF = spark.createDataFrame(teams, teamsSchema)
+```
+
+## Join Type 1: Inner Joins
+
+If we wanted to know the team names for every kid in our dataset, we would do the following:
+
+```scala
+val joinCondition = kidsDF.col("Team") === teamsDF.col("TeamId")
+val kidsTeamsDF = kidsDF.join(teamsDF, joinCondition, "inner")
+```
+
+which would give us the following table if we showed the new DataFrame:
+
+```
++---+-------+----+------+---------------+
+| Id| Name|Team|TeamId| TeamName|
++---+-------+----+------+---------------+
+| 40| Mary| 1| 1|The Invincibles|
+| 44|Charlie| 1| 1|The Invincibles|
+| 41| Jane| 3| 3| Rockstars|
+| 42| David| 3| 3| Rockstars|
+| 43| Angela| 2| 2| Dog Lovers|
+| 45| Jimmy| 2| 2| Dog Lovers|
++---+-------+----+------+---------------+
+```
+
+The above join we showed above is the "natural" kind of join we would expect when we combine two different tables - we'll keep just the rows from both tables that match the join condition, and we'll form a new table out of those combined rows. This is called an **inner** join, and it's the default join type Spark will perform... that is, unless we specify another kind of join.
+
+This is where the fun starts.
+
+## Join Type 2: Outer Joins
+
+Assuming we still want to attach a team to every kid, the inner join above will completely omit the Lonely kid in our DataFrame, because this poor kid doesn't have a team. We would be able to show this kid in the resulting table by placing a null next to it, so that the class teacher can spot poor Lonely and assign them a team.
+
+A left-outer join does that. All the rows in the left/first DataFrame will be kept, and wherever a row doesn't have any corresponding row on the right (the argument to the `join` method), we'll just put nulls in those columns:
+
+```scala
+kidsDF.join(teamsDF, joinCondition, "left_outer")
+```
+
+Notice the `"left_outer""` argument there. This will print the following table:
+
+```
++---+-------+----+------+---------------+
+| Id| Name|Team|TeamId| TeamName|
++---+-------+----+------+---------------+
+| 40| Mary| 1| 1|The Invincibles|
+| 44|Charlie| 1| 1|The Invincibles|
+| 41| Jane| 3| 3| Rockstars|
+| 42| David| 3| 3| Rockstars|
+| 46| Lonely| 7| null| null|
+| 43| Angela| 2| 2| Dog Lovers|
+| 45| Jimmy| 2| 2| Dog Lovers|
++---+-------+----+------+---------------+
+```
+
+Now poor Lonely has null where the team details are supposed to be shown.
+
+If we wanted to do the reverse - show all the teams which have no members, we would do a `right_outer` join. Same principle:
+
+```scala
+kidsDF.join(teamsDF, joinCondition, "left_outer")
+```
+
+(notice the argument to the method there) and the output would be:
+
+```
++----+-------+----+------+--------------------+
+| Id| Name|Team|TeamId| TeamName|
++----+-------+----+------+--------------------+
+| 40| Mary| 1| 1| The Invincibles|
+| 44|Charlie| 1| 1| The Invincibles|
+| 41| Jane| 3| 3| Rockstars|
+| 42| David| 3| 3| Rockstars|
+|null| null|null| 4|The Non-Existent ...|
+| 43| Angela| 2| 2| Dog Lovers|
+| 45| Jimmy| 2| 2| Dog Lovers|
++----+-------+----+------+--------------------+
+```
+
+Notice how the Non-Existent Team has no members, so it appears once in the table with `null` where a kid is supposed to be.
+
+If we wanted to show both kids that have no teams AND teams that have no kids, we can get a combined result by using an `outer` join:
+
+```
+kidsDF.join(teamsDF, joinCondition, "outer")
+```
+
+which gives us
+
+```
++----+-------+----+------+--------------------+
+| Id| Name|Team|TeamId| TeamName|
++----+-------+----+------+--------------------+
+| 40| Mary| 1| 1| The Invincibles|
+| 44|Charlie| 1| 1| The Invincibles|
+| 41| Jane| 3| 3| Rockstars|
+| 42| David| 3| 3| Rockstars|
+|null| null|null| 4|The Non-Existent ...|
+| 43| Angela| 2| 2| Dog Lovers|
+| 45| Jimmy| 2| 2| Dog Lovers|
+| 46 | Lonely| 7| null| null|
++----+-------+----+------+--------------------+
+```
+
+You've probably encountered these concepts from standard databases. We use inner joins and outer joins (left, right or both) ALL the time. However, this is where the fun starts, because Spark supports more join types. Let's have a look.
+
+## Join Type 3: Semi Joins
+
+Semi joins are something else. Semi joins take all the rows in one DF such that _there is a row on the other DF so that the join condition is satisfied_. In other words, it's essentially a filter based on the existence of a matching key on the other DF. In SQL terms, we can express this computation as `WHERE EXISTS (SELECT * FROM otherTable WHERE joinCondition)`.
+
+For our use case, a left semi join will show us all kids which have a team:
+
+```scala
+kidsDF.join(teamsDF, joinCondition, "left_semi").show
+```
+
+and we would show this:
+
+```
++---+-------+----+
+| Id| Name|Team|
++---+-------+----+
+| 40| Mary| 1|
+| 44|Charlie| 1|
+| 41| Jane| 3|
+| 42| David| 3|
+| 43| Angela| 2|
+| 45| Jimmy| 2|
++---+-------+----+
+```
+
+As expected, Lonely is not here.
+
+## Join Type 4: Anti Joins
+
+Anti joins are also very interesting. They're essentially the opposite of semi joins: they return all the rows from one table such that _there is NO row on the other table satisfying the join condition_. In SQL terms, this is equivalent with `WHERE NOT EXISTS (SELECT * FROM otherTable WHERE joinCondition)`.
+
+In our case, a left anti join would show all kids who do NOT have a team yet:
+
+```scala
+kidsDF.join(teamsDF, joinCondition, "left_anti").show
+```
+
+As expected, Lonely should show up here:
+
+```
++---+------+----+
+| Id| Name|Team|
++---+------+----+
+| 46|Lonely| 7|
++---+------+----+
+```
+
+## Join type 5: Cross Joins
+
+A cross join describes all the possible combinations between two DFs. Every one is game. Here's how we can do it:
+
+```scala
+kidsDF.crossJoin(teamsDF)
+```
+
+This would produce the quite-big-for-small-data table:
+
+```
++---+-------+----+------+--------------------+
+| Id| Name|Team|TeamId| TeamName|
++---+-------+----+------+--------------------+
+| 40| Mary| 1| 1| The Invincibles|
+| 40| Mary| 1| 2| Dog Lovers|
+| 40| Mary| 1| 3| Rockstars|
+| 40| Mary| 1| 4|The Non-Existent ...|
+| 41| Jane| 3| 1| The Invincibles|
+| 41| Jane| 3| 2| Dog Lovers|
+| 41| Jane| 3| 3| Rockstars|
+| 41| Jane| 3| 4|The Non-Existent ...|
+| 42| David| 3| 1| The Invincibles|
+| 42| David| 3| 2| Dog Lovers|
+| 42| David| 3| 3| Rockstars|
+| 42| David| 3| 4|The Non-Existent ...|
+| 43| Angela| 2| 1| The Invincibles|
+| 43| Angela| 2| 2| Dog Lovers|
+| 43| Angela| 2| 3| Rockstars|
+| 43| Angela| 2| 4|The Non-Existent ...|
+| 44|Charlie| 1| 1| The Invincibles|
+| 44|Charlie| 1| 2| Dog Lovers|
+| 44|Charlie| 1| 3| Rockstars|
+| 44|Charlie| 1| 4|The Non-Existent ...|
++---+-------+----+------+--------------------+
+```
+
+So as you can see, the size of your resulting data simply explodes. Being a cartesian product, the size of the resulting DF is the product of the individual sizes of the joined DFs. So be careful in production - I'd generally avoid cross joins unless absolutely necessary.
+
+## Conclusion
+
+There you have it, folks: all the join types you can perform in Apache Spark. Even if some join types (e.g. inner, outer and cross) may be quite familiar, there are some interesting join types which may prove handy as filters (semi and anti joins).
diff --git a/_posts/2020-10-23-stateful-stateless-actors.md b/_posts/2020-10-23-stateful-stateless-actors.md
new file mode 100644
index 000000000000..caa574349d79
--- /dev/null
+++ b/_posts/2020-10-23-stateful-stateless-actors.md
@@ -0,0 +1,169 @@
+---
+title: "Akka Typed Actors: Stateful and Stateless"
+date: 2020-10-22
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka]
+excerpt: "Akka Typed has fundamentally changed the way we create actors. In this article we'll look at several ways we can keep state inside Akka actors."
+---
+
+This article is for people getting started with Akka typed actors. We'll look at how we can keep state inside an actor in a few different ways.
+
+## Setup
+
+This article will assume you have the Akka Typed library set up in your project. If you don't, you can create a new SBT project with your favorite tool, and add the following to your build.sbt file:
+
+```scala
+val akkaVersion = "2.6.10"
+
+libraryDependencies += "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion
+```
+
+## Background
+
+This article assumes you know the principles of Akka actors. In short:
+
+- standard multithreading/parallel applications are a pain to write because of concurrency issues
+- in Akka, we design applications in terms of actors
+- an actor is an object whose state we cannot access directly, but we can only interact with it via asynchronous messages
+- message passing and handling eliminates the need for us to manage threads & concurrency, while making it easy to write massively distributed systems
+
+An actor is described by its behavior, which (among other things) is responsible for handling the messages that the actor can receive. After each message, the actor's behavior can change: given new information, the actor might change the way it handles future messages - much like us humans in real life.
+
+An important part of an actor is the potential data it might hold - we call that its _state_. As a reaction to an incoming message, the data held inside the actor might change.
+
+As an aside, this actor model embodies the real encapsulation principle: you can never access the internal state of the actor. We can never call some `getState()` method on it, but we can only interact with it via asynchronous messages. This is what object-oriented programming was [supposed to be](http://userpage.fu-berlin.de/~ram/pub/pub_jf47ht81Ht/doc_kay_oop_en) (search for "messages"). For some reason, OOP took a different turn... but I digress.
+
+## A Stateful Emotional Actor
+
+For this example, we'll write an actor that reacts to external messages from the world by changing its happiness level, originally starting at zero. Let's assume we have a few message types to send to this actor:
+
+```scala
+trait SimpleThing
+case object EatChocolate extends SimpleThing
+case object WashDishes extends SimpleThing
+case object LearnAkka extends SimpleThing
+```
+
+We can then define an actor with a mutable piece of data (state) as follows:
+
+```scala
+import akka.actor.typed.Behavior
+import akka.actor.typed.scaladsl.Behaviors
+
+val emotionalMutableActor: Behavior[SimpleThing] = Behaviors.setup { context =>
+ // define internal state
+ var happiness = 0
+
+ Behaviors.receiveMessage {
+ case EatChocolate =>
+ context.log.info(s"($happiness) Eating chocolate, getting a shot of dopamine!")
+ // change internal state
+ happiness += 1
+ // new behavior for future messages
+ Behaviors.same
+ // similar cases for the other messages
+ case WashDishes =>
+ context.log.info(s"($happiness) Doing chores, womp, womp...")
+ happiness -= 2
+ Behaviors.same
+ case LearnAkka =>
+ context.log.info(s"($happiness) Learning Akka, looking good!")
+ happiness += 100
+ Behaviors.same
+ case _ =>
+ context.log.warn(s"($happiness) Received something i don't know")
+ Behaviors.same
+ }
+ }
+```
+
+In order to use mutable state, we create this behavior using `Behaviors.setup`, which allows you to allocate resources at the moment of instantiation, before any messages can arrive to this actor.
+
+If we want to test this actor, all we have to do is back it up by an `ActorSystem` and then fire a few messages to see how it does:
+
+```scala
+ def demoActorWithState(): Unit = {
+ val emotionalActorSystem = ActorSystem(emotionalMutableActor, "EmotionalSystem")
+
+ emotionalActorSystem ! EatChocolate
+ emotionalActorSystem ! EatChocolate
+ emotionalActorSystem ! WashDishes
+ emotionalActorSystem ! LearnAkka
+
+ Thread.sleep(1000)
+ emotionalActorSystem.terminate()
+ }
+```
+
+So if we call this in our main method and run our application, we'll get the following log lines:
+
+```
+[2020-10-22 17:49:32,854] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (0) Eating chocolate, getting a shot of dopamine!
+[2020-10-22 17:49:32,854] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (1) Eating chocolate, getting a shot of dopamine!
+[2020-10-22 17:49:32,854] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (2) Doing chores, womp, womp...
+[2020-10-22 17:49:32,854] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (0) Learning Akka, looking good!
+```
+
+So we see that with every new message, our actor modified its internal state.
+
+Mutable variables are generally fine inside an actor, because handling a message is thread-safe*, so we can safely change our variable without worrying that some other thread might race to change or read that variable at the same time.
+
+\* except Future callbacks inside actors, which will be a discussion for another time...
+
+## A Stateless Emotional Actor
+
+But in pure Scala we hate variables and anything mutable. So in this part, I'll show you how we can write the same actor without needing a variable. Instead of a variable, we'll use a method taking an `Int` argument and returning a `Behavior` instance:
+
+```scala
+def emotionalFunctionalActor(happiness: Int = 0): Behavior[SimpleThing] = Behaviors.receive { (context, message) =>
+ // handle message here
+}
+```
+
+Notice that we moved our mutable variable as a method argument, and because we don't have a variable to initialize, we don't need `Behaviors.setup` anymore, so we can directly use `Behaviors.receive`. Inside the block, all we have to do is run a pattern match on the message and do something similar to what we did earlier. However, this time, we aren't returning `Behaviors.same` on every branch, but rather a new behavior obtained by calling `emotionalFunctionalActor` with a new value for happiness:
+
+```scala
+ def emotionalFunctionalActor(happiness: Int = 0): Behavior[SimpleThing] = Behaviors.receive { (context, message) =>
+ message match {
+ case EatChocolate =>
+ context.log.info(s"($happiness) eating chocolate")
+ // change internal state
+ emotionalFunctionalActor(happiness + 1)
+ case WashDishes =>
+ context.log.info(s"($happiness) washing dishes, womp womp")
+ emotionalFunctionalActor(happiness - 2)
+ case LearnAkka =>
+ context.log.info(s"($happiness) Learning Akka, yes!!")
+ emotionalFunctionalActor(happiness + 100)
+ case _ =>
+ context.log.warn("Received something i don't know")
+ Behaviors.same
+ }
+ }
+```
+
+Sure enough, if we change our test method to use this new `emotionalFunctionalActor` instead, the logged output will look the same:
+
+```
+[2020-10-22 17:59:27,334] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (0) eating chocolate
+[2020-10-22 17:59:27,335] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (1) eating chocolate
+[2020-10-22 17:59:27,335] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (2) washing dishes, womp womp
+[2020-10-22 17:59:27,335] [INFO] [live.day2actors.AkkaEssentials$] [] [EmotionalSystem-akka.actor.default-dispatcher-3] - (0) Learning Akka, yes!!
+```
+
+## How to Turn a Stateful Actor into Stateless
+
+Here are some steps to turn a stateful actor — with variables or mutable pieces of data — into a "stateless" actor:
+
+1. Create your actor behavior as a method. The arguments of the method will be immutable versions of the pieces of data you used to hold.
+2. If you created your stateful actor with `Behaviors.setup`, you'll probably no longer need it — use `Behaviors.receive` or `Behaviors.receiveMessage`.
+3. Most of the time, stateful actors keep the same behavior after the reception of a message — see earlier case where we returned `Behaviors.same` every time. This time, with every message reception, you'll change the behavior to a new method call with new arguments, depending on the data you need to change.
+
+You may be wondering whether calling the said method again in the message handling cases (this case `emotionalFunctionalActor`) can blow up the stack. This is an interesting topic.
+
+This "recursive" method call is not truly recursive. Remember, when a thread schedules this actor for execution, it will dequeue messages off its mailbox. Once it handles a message, it will create a new behavior which it will attach to the actor — of course, by calling the `emotionalFunctionalActor` or whatever your method is. But this method returns _immediately_ with a new behavior — it won't call itself forever, the thread just calls it once, and it returns an object. Once the actor is again scheduled for execution — perhaps even on the same thread as before — the thread will simply apply that behavior on the message again, create a new behavior, etc. Nothing truly recursive happens there, because the behavior is invoked at a different time.
+
+## Conclusion
+
+We've seen how we can create stateful actors in Akka Typed, how a "stateless" actor looks like, and how to turn mutable state into method arguments in a stateless actor version. I hope this is useful, and that you'll create more functional-style/"stateless" actors after this article!
diff --git a/_posts/2020-10-26-pipe-pattern.md b/_posts/2020-10-26-pipe-pattern.md
new file mode 100644
index 000000000000..de1aaf35181c
--- /dev/null
+++ b/_posts/2020-10-26-pipe-pattern.md
@@ -0,0 +1,183 @@
+---
+title: "Akka Typed: How the Pipe Pattern Prevents Anti-Patterns"
+date: 2020-10-26
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka]
+excerpt: "Akka Typed has not only made fundamental changes in the actor protocol definitions, but made significant improvements in actor mechanics as well."
+---
+
+This article is for people who are getting familiar with Akka Typed actors. You don't have to be an expert — that would certainly be a plus — but some familiarity with actor concepts is assumed.
+
+## Setup
+
+We assume you have Akka Typed in your project. If not, just create a new SBT project and add the following to your build.sbt:
+
+```scala
+val akkaVersion = "2.6.10"
+
+libraryDependencies += "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion
+```
+
+## Background
+
+This piece assumes you know the first principles of Akka actors (check the intro of [this article](/stateful-stateless-actors) for an introduction). In particular for this article, we care most about actor **encapsulation**: the state of an actor is inaccessible from the outside, even in a multithreaded/distributed environment. We can only communicate with an actor via message exchanges.
+
+However, in "real life", our actor may not necessarily block on resources while handling a message. We often make our actors interact with otherwise asynchronous services. These asynchronous services can break actor encapsulation, because handling an asynchronous response happens on some thread — potentially a different thread than the one that just took control of the actor.
+
+## An Anti-Pattern
+
+Imagine we're designing a Twilio-like service which performs phone calls. To call somebody (a customer, a friend, etc) we have a "database" of name-number pairs that we can access via an asynchronous call. The "infrastructure", in this simplified model, looks like this:
+
+```scala
+import scala.concurrent.{ExecutionContext, Future}
+import java.util.concurrent.Executors
+
+object Infrastructure {
+
+ private implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(8))
+
+ private val db: Map[String, Int] = Map(
+ "Daniel" -> 123,
+ "Alice" -> 456,
+ "Bob" -> 999
+ )
+
+ // external API
+ def asyncRetrievePhoneNumberFromDb(name: String): Future[Int] =
+ Future(db(name))
+}
+```
+
+Assume we're designing an actor which can receive a command to initiate a phone call to a person. This actor would call the external service, and upon obtaining the phone number, it would initiate the call. A quick implementation would look like this:
+
+```scala
+import akka.actor.typed.scaladsl.{Behaviors, Routers}
+import scala.util.{Failure, Success}
+
+trait PhoneCallProtocol
+case class FindAndCallPhoneNumber(name: String) extends PhoneCallProtocol
+
+val quickPhoneCallInitiator: Behavior[PhoneCallProtocol] =
+ Behaviors.setup { (context, message) =>
+ var nPhoneCalls = 0
+ var nFailures = 0
+
+ Behaviors.receiveMessage {
+ case FindAndCallPhoneNumber(name) =>
+ val futureNumber: Future[Int] = asyncRetrievePhoneNumberFromDb(name)
+ futureNumber.onComplete {
+ case Success(number) =>
+ // actually perform the phone call here
+ context.log.info(s"Initiating phone call to $number")
+ nPhoneCalls += 1 // please cringe here
+ case Failure(ex) =>
+ context.log.error(s"Phone call to $name failed: $ex")
+ nFailures += 1 // please cringe here
+ }
+ Behaviors.same
+ }
+ }
+```
+
+After designing the actor protocol in terms of the commands it can receive (here, only one), we are setting up the actor state with `Behaviors.setup` and then returning a message handler with `Behaviors.receive`. In this handler, upon receiving the `FindAndCallPhoneNumber` command, this actor would invoke the external service, then process the resulting future with `.onComplete`.
+
+So what's the problem?
+
+`Future` callbacks, as well as transformations, are evaluated on _some_ thread. This thread may or may not be the one that's handling the message. In other words, each line with "please cringe here" is a race condition. **We've broken the actor encapsulation.**
+
+A second drawback is that, since changing actor state happens in a `Future` callback, we can't make this actor [stateless](/stateful-stateless-actors).
+
+## Enter Pipes
+
+There is another way which is completely safe, both from a type perspective and from a multithreading perspective.
+
+The question is: why handle the `Future` manually at all? Why not send the result of that `Future` to this actor as a message, which it can later handle in a thread-safe way?
+
+This technique is the pipe pattern. We are going to automatically redirect the contents of the `Future` back to this actor, as a message which it will receive later. There are two important aspects to this approach:
+
+- A benefit: no more encapsulation break, since handling the result of the `Future` will be handled as a message in a thread-safe way
+- A benefit + responsibility: because the actor is typed, we need to transform the result of the `Future` (which is a `Try[Something]`) into a message type the actor supports
+
+In order to make pipes work, the result of our "infra" asynchronous call (either successful or failed) needs to be transformed into a message type the actor supports, so we'll need to create two more message classes:
+
+```scala
+case class InitiatePhoneCall(number: Int) extends PhoneCallProtocol
+case class LogPhoneCallFailure(reason: Throwable) extends PhoneCallProtocol
+```
+
+After which we can make the actor send the future result to itself later, and handle the new messages:
+
+```scala
+val phoneCallInitiatorV2: Behavior[PhoneCallProtocol] =
+ Behaviors.setup { (context, message) =>
+ var nPhoneCalls = 0
+ var nFailures = 0
+
+ Behaviors.receiveMessage {
+ case FindAndCallPhoneNumber(name) =>
+ val futureNumber: Future[Int] = asyncRetrievePhoneNumberFromDb(name)
+ // pipe makes all the difference
+ // transform the result of the future into a message
+ context.pipeToSelf(futureNumber) {
+ case Success(phoneNumber) =>
+ // messages that will be sent to myself
+ InitiatePhoneCall(phoneNumber)
+ case Failure(ex) =>
+ LogPhoneCallFailure(ex)
+ }
+ Behaviors.same
+ case InitiatePhoneCall(number) =>
+ // perform the phone call
+ context.log.info(s"Starting phone call to $number")
+ nPhoneCalls += 1 // no more cringing
+ Behaviors.same
+ case LogPhoneCallFailure(ex) =>
+ context.log.error(s"Calling number failed: $ex")
+ nFailures += 1 // no more cringing
+ Behaviors.same
+ }
+ }
+```
+
+Notice the `pipeToSelf` call. We pass a `Future` and a function which transforms a `Try[Int]` into a message this actor will handle later. In the message handlers, we are then free to change actor state, because handling a message is atomic. We've repaired the actor encapsulation.
+
+This pattern now enables us to make the actor stateless if we wanted, because changing state happens in a message handler. So we can further refactor our actor:
+
+```scala
+def phoneCallInitiatorV3(nPhoneCalls: Int = 0, nFailures: Int = 0): Behavior[PhoneCallProtocol] =
+ Behaviors.receive { (context, message) =>
+ message match {
+ case FindAndCallPhoneNumber(name) =>
+ val futureNumber: Future[Int] = asyncRetrievePhoneNumberFromDb(name)
+ // pipe makes all the difference
+ // transform the result of the future into a message
+ context.pipeToSelf(futureNumber) {
+ case Success(phoneNumber) =>
+ // messages that will be sent to myself
+ InitiatePhoneCall(phoneNumber)
+ case Failure(ex) =>
+ LogPhoneCallFailure(ex)
+ }
+ Behaviors.same
+ case InitiatePhoneCall(number) =>
+ // perform the phone call
+ context.log.info(s"Starting phone call to $number")
+ // change behavior
+ phoneCallInitiatorV3(nPhoneCalls + 1, nFailures)
+ case LogPhoneCallFailure(ex) =>
+ // log failure
+ context.log.error(s"Calling number failed: $ex")
+ // change behavior
+ phoneCallInitiatorV3(nPhoneCalls, nFailures + 1)
+ }
+ }
+```
+
+Notice how we turned the `val` behavior into a `def` which now keeps the "state" as method arguments. Wherever we used to change state, now we return a new behavior containing the new "state" as method arguments. Because there's nothing mutable to set up, we don't need `Behaviors.setup` and now use `Behaviors.receiveMessage` instead.
+
+To end, the `pipeToSelf` call is completely thread-safe and fine to call even from other Future callbacks.
+
+## Conclusion
+
+In this article, we explored how the pipe pattern solves a potentially serious problem when handling `Futures` inside an actor's scope, how we can repair it and (as a bonus) how we can make an actor stateless even while handling results from external services. Hopefully this is useful!
diff --git a/_posts/2020-11-02-scala-3-indentation.md b/_posts/2020-11-02-scala-3-indentation.md
new file mode 100644
index 000000000000..0bcf0e09dbfd
--- /dev/null
+++ b/_posts/2020-11-02-scala-3-indentation.md
@@ -0,0 +1,187 @@
+---
+title: "Let's Talk About the Scala 3 Indentation"
+date: 2020-11-02
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala 3]
+excerpt: "Some people love it, some hate it. Scala 3 indented syntax is not that bad, and it might actually help. Let's take a look at how Scala 3 can change the structure of our code."
+---
+
+Regardless of whether you're experienced or new to Scala, you've probably been confused about Scala syntax inconsistencies and alternatives at least once. In this article, we'll take a careful, structured look at how Scala 3 adds yet another facility to our ever-expanding set of alternative code styles.
+
+This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## 1. If Expressions
+
+The Scala 2 syntax allows you to write if expressions on multiple lines:
+
+```scala3
+val aCondition = if (2 > 3)
+ "bigger"
+ else "smaller"
+```
+
+Of course, that's not how aesthetics are generally chosen. Some styles include:
+
+```scala3
+// one-liner
+val aCondition = if (2 > 3) "bigger" else "smaller"
+
+// java-style
+val aCondition2 =
+ if (2 > 3) {
+ "bigger"
+ } else {
+ "smaller"
+ }
+
+// compact
+val aCondition3 =
+ if (2 > 3) "bigger"
+ else "smaller"
+```
+
+To be clear, all the above _are still supported_ in Scala 3. However, we can make do without the braces and even the parentheses of if expressions, while _indentation becomes significant_.
+
+```scala3
+val anIndentedCondition =
+ if 2 > 3
+ "bigger"
+ else
+ "smaller"
+```
+
+When we remove the parens around `2 > 3`, indentation becomes significant for this expression. This means that the `else` branch must be at least at the indent level of the `if`. In real life, mixed indentations for if/else branches (especially in junior/hacked code) are extremely confusing especially in chained conditions, so the compiler helps in this case. The code also looks a bit cleaner than the Java-style version above.
+
+If we want to place the if-branch result on the same line as the condition, we need to add a `then` keyword:
+
+```scala3
+// one-liner
+val aCondition = if 2 > 3 then "bigger" else "smaller"
+// compact
+val aCondition =
+ if 2 > 3 then "bigger"
+ else "smaller"
+```
+
+That's it!
+
+## 2. For Comprehensions
+
+A similar change has been added to other control structures like `for` comprehensions. The idea is: if we don't add braces, indentation becomes significant. Here's what we used to write:
+
+```scala3
+for {
+ n <- List(1, 2, 3)
+ c <- List('a', 'b', 'c')
+} yield s"$c$n"
+```
+
+Now, in Scala 3:
+
+```scala3
+for
+ n <- List(1, 2, 3)
+ c <- List('a', 'b', 'c')
+yield s"$c$n"
+```
+
+Same code, right? That's because the Scala 2 version also included some proper aesthetics. Without braces, those indents are now mandatory.
+
+No biggie.
+
+## 3. Indentation regions
+
+In the syntactic analysis compilation step, the compiler adds indentation regions after certain tokens. Translation: as per the [official docs](http://dotty.epfl.ch/docs/reference/other-new-features/indentation.html#optional-braces), after the keywords `if then else while do try catch finally for yield match return` and the tokens `= => <-`, we can break the line and write our code one level deeper than the line above. This indentation level will serve as a baseline for the other expressions that we might nest inside. The compiler does it by adding some fake tokens at line breaks (`` or ``) to keep track of the indentation level without multiple passes over the code.
+
+This means that methods can now be implemented without braces:
+
+```scala3
+def computeMeaningOfLife(year: Int): Int =
+ println("thinking...")
+
+ 42 // <-- indent matters, so it's taken under the scope of this method
+```
+
+This part may be particularly confusing. The way I like to talk about it is: imagine the compiler inserted braces between `=` and your returned value; in this way, the implementation of the method is a _code block_, which, obviously, is a single expression whose value is given by its last constituent expression. The significant indentation means, in this case, that we actually have an invisible code block there.
+
+An indentation region is also created when we define classes, traits, objects or [enums](https://blog.rockthejvm.com/enums-scala-3/) followed by a colon `:` and a line break. This token is now interpreted by the compiler as "colon at end of line", which is to say "colon then define everything indented". Examples:
+
+```scala3
+class Animal:
+ def eat(): Unit
+
+trait Carnivore:
+ def eat(animal: Animal): Unit
+
+object Carnivore:
+ def apply(name: String): Carnivore = ???
+```
+
+Similar rules apply for extension methods and given instances (we'll talk about them in a later article):
+
+```scala3
+given myOrder as Ordering[Int]: // <-- start the indentation region
+ def compare(x: Int, y: Int) =
+ if x < y then 1 // notice my new syntax
+ else if x > y then -1
+ else 0
+```
+
+Now for the million dollar question: indent with _spaces or tabs_? Scala 3 supports both, and the compiler is able to compare indentations.
+
+If two lines start with the same number of spaces or the same number of tabs, the indentations are comparable, and the comparison is given by the number of spaces/tabs after. For example, 3 tabs + one space is "less indented" than 3 tabs + 2 spaces. Similarly, 4 spaces + 1 tab is "less indented" than 4 spaces + 2 tabs. Makes sense. If two lines don't start with the same number of spaces/tabs, they are _incomparable_, e.g. 3 tabs vs 6 spaces. The compiler always knows the indentation baseline for an indentation region, so if a line is incomparable with the baseline, it'll give an error.
+
+Here's the bottom line: just don't mix spaces with tabs. Pick your camp, fight to the death, but don't mix 'em.
+
+## 4. Function arguments
+
+There's a common style of writing Scala 2 with one-arg methods:
+
+```scala3
+val aFuture = Future {
+ // some code
+}
+```
+
+which is the same as `Future(/* that block of code */)`, which throws some newcomers off. This brace syntax is applicable to any method with a single argument.
+
+Here's the thing: this one is here to stay. No significant indentation here. We could add support for it with the `-Yindent-colons` compiler option, which would allow us to add an end-of-line `:` and write the method argument indented:
+
+```scala3
+val nextYear = Future:
+ 2021
+```
+
+or
+
+```scala3
+List(1,2,3).map:
+ x => x + 1
+```
+
+However, this is not part of the core language rules.
+
+## 5. The `end` Game
+
+This new concept of indentation regions can cause confusions with large blobs of code, particularly in the class definition department - we tend to write lots of code there. Even chained if-expressions can also become hard to read while indented, since code for branches may span several lines, sometimes with whitespace in between them, so it's hard to pinpoint exactly where each code belongs.
+
+To that end (pun intended), Scala 3 introduced the `end` token to differentiate which code belongs to which indentation region:
+
+```scala3
+class Animal:
+ def eat(): Unit =
+ if System.currentTimeMillis() % 2 == 0
+ println("even")
+ else
+ println("odd")
+ end if
+ end eat
+end Animal
+```
+
+Obviously, this example is trivial, but the `end` token will definitely prove useful when we have 100x more code in the same file than in the above snippet, when we have lots of indents, and/or lots of whitespace in between lines. The `end` token does not influence the compiler, but it was added for our ease of reading code.
+
+## 6. Conclusion
+
+There's not much more to it - pretty much everything you need to know about indentation with Scala 3. When I personally looked at the new indentation rules for Scala 3, I personally thought, "what have you done?!". Come to take a more structured approach, it's not that bad, and it might actually help. Several people already report that Scala feels faster to write, easier to read and generally more productive with this style. Only time will tell - in any event, if this article made even one person think, "this isn't as bad as I thought", then it was a success!
diff --git a/_posts/2020-11-10-scala-variance-positions.md b/_posts/2020-11-10-scala-variance-positions.md
new file mode 100644
index 000000000000..6785bc9ae4ed
--- /dev/null
+++ b/_posts/2020-11-10-scala-variance-positions.md
@@ -0,0 +1,267 @@
+---
+title: "Variance Positions in Scala, Demystified"
+date: 2020-11-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, type system]
+excerpt: "We'll take a look at the famous \"covariant type occurs in contravariant position\" problem, and what we can do about it."
+toc: true
+toc_label: "In this article"
+---
+
+Variance in Scala is one of the most powerful concepts of its type system. It's simple in theory, but it can cause massive headaches when we think we got the hang of writing abstract, generic code. In this article, we'll take a careful and structured look at one of the most cryptic compiler errors you'll ever see:
+
+Error: covariant type T occurs in contravariant position in type T of value myArg
+{: .notice--danger}
+
+We won't need a particular project setup for this article - a plain Scala/SBT project will do.
+
+> We discuss variance and variance positions in depth in the [Advanced Scala](https://rockthejvm.com/p/advanced/scala) course. Check it out!
+## 1. Background
+
+This article will assume you're familiar with generics (either in Java, Scala or some other statically typed language).
+
+Beyond the capability of the Java generics, Scala also has the concept of _variance_, which is a way of transferring the subtype relationship from the type arguments to the generic type.
+
+This sounds quite complicated until we bring it back to real life. Imagine we're implementing a generic collection which we'll call `MyList[T]`. If we have a small class hierarchy — -e.g. the classical example with `Animal`s and then `Cat`s and `Dog`s which extend Animals — then it makes sense to ask the following question:
+
+> If Dogs are also Animals, then is a list of Dogs also a list of Animals?
+
+For a list, the answer is yes. However, this answer doesn't make sense for every type.
+
+## 2. The Variance Question
+
+To put it another way, if we wanted to write a new `Thing[T]` generic type in our code base, the following question is important:
+
+> If A is a subtype of B, then should Thing[A] be a subtype of Thing[B]?
+
+This is the _variance question_.
+
+If our `Thing` is a collection, such as a list, the answer is yes: if Dogs are Animals, then a collection of Dogs is also a collection of Animals. So from a substitution perspective, we could assign a collection of Dogs to a collection of Animals:
+
+```scala
+val dogs: MyList[Animal] = new MyList[Dog]
+```
+
+We say that MyList is _covariant_ in its type argument [T], and we mark it with a small `+` sign in the class declaration: `class MyList[+T]`
+
+But that's not the only possibility. We can also answer "no", i.e. Thing[A] and Thing[B] have no subtype relationship, i.e. we can't assign one to a value bearing the other type. In this case, we'd call Thing _invariant_ in its type argument [T], and we leave the type argument T as it is in the class declaration, i.e. `class Thing[T]`
+
+There's still one more possible answer: "hell no, it's backwards!". In other words, if A is a subtype of B, _then `Thing[B]` is a subtype of `Thing[A]`_. This is called [contravariant](/contravariance), and we mark it with a `-` sign at the class declaration. For the Animals use case, a good contravariant example would be a Vet:
+
+```scala
+class Vet[-T]
+
+val lassiesVet: Vet[Dog] = new Vet[Animal]
+```
+
+A Vet[Animal] is a proper replacement for a Vet[Dog], because a Vet can treat any Animal, and so she/he can treat a Dog too.
+
+If you want to keep a good rule of thumb on how to pick variance for your new generic type, it's this:
+
+
+
+## 3. The Variance Positions Problem
+
+Now that we've found that a collection should be covariant, we get to work and write our own list, because of course we can:
+
+```scala
+abstract class MyList[+T] {
+ def head: T
+ def tail: MyList[T]
+ def add(elem: T): MyList[T]
+}
+```
+
+We start from the very basics. But before we even write a proper subtype for MyList, we hit a wall:
+
+Error: covariant type T occurs in contravariant position in type T of value elem
+{: .notice--danger}
+
+Just as we thought we understood variance, here comes the compiler trying to prove we don't know squat.
+
+We'll take a careful, structured approach of what this problem means, and then we'll learn how we can solve it. For the following examples, we'll use the same example with animals.
+
+### 3.1. Types of `val`s Are in Covariant Position
+
+Let's say we had a Vet. As discussed before, a Vet should be a contravariant type. Let's also imagine this vet had a favorite animal `val` field, of the same type she can treat:
+
+```scala
+class Vet[-T](val favoriteAnimal: T)
+```
+
+Assuming this was possible, then the following code would compile:
+
+```scala
+val garfield = new Cat
+val theVet: Vet[Animal] = new Vet[Animal](garfield)
+val lassiesVet: Vet[Dog] = theVet
+```
+
+See any trouble here?
+
+- `lassiesVet` is declared as `Vet[Dog]`; as per contravariance we can assign a `Vet[Animal]`
+- `theVet` is a `Vet[Animal]` (typed correctly), but is constructed with a `Cat` (a legit Animal)
+- `lassiesVet.favoriteAnimal` is supposed to be a `Dog` per the generic type declared, but it's really a `Cat` per its construction
+
+No-no. This is a type conflict. A sound type checker will not compile this code. **We say that the types of `val` fields are in _covariant_ position**, and this is what the compiler will show you:
+
+Error: contravariant type T occurs in covariant position in type T of value favoriteAnimal
+{: .notice--danger}
+
+If the generic type was covariant (e.g. a list), then it would work. If the generic type was invariant, we wouldn't even have this problem, as we'd have the same type T everywhere.
+
+### 3.2. Types of `var`s Are Also in Contravariant Position
+
+If our contravariant Vet example had a `var` field, we'd have the exact same problem right at initialization. Therefore, **types of `var` members are in _covariant_ position** as well. Because `var`s can be reassigned, they come with an extra restriction.
+
+Let's think about `Option[T]` for a second. Should it be covariant or contravariant?
+
+Spoiler: it's covariant. If `Dog` extends `Animal` then `Option[Dog]` should be a subtype of `Option[Animal]`. Pick your favorite reason (both are true):
+
+- If a dog is an animal, then a maybe-dog is also a maybe-animal.
+- Think of an Option as a list with at most one element. If a list is covariant, then an option is covariant.
+
+Now, imagine that (for whatever reason) we had a mutable version of an option. Let's call it `MutableOption[+T]` with a subtype containing a `var` member, e.g. `class MutableSome[+T](var contents: T)`.
+
+Here's what we're going to do next:
+
+```scala
+val maybeAnimal: MutableSome[Animal] = new MutableSome[Dog](new Dog)
+maybeAnimal.contents = new Cat
+```
+
+The first line is perfectly legal: because `MutableSome` is covariant, we can assign a maybe-dog on the right-hand side of the assignment. Now, because we declared `maybeAnimal` to be a `MutableSome[Animal]`, the compiler would allow us to change the `contents` variable to another kind of `Animal`. Because a `Cat` is an `Animal`, then a `Cat` is a legal value to use, which would also blow up the type guarantee. That is because the original `MutableSome[Dog]` we used on the first line also comes with a guarantee that the contents are of type `Dog`, but we've just ruined it.
+
+Therefore, we say that **types of `var` fields are in _contravariant_ position**.
+
+But wait, didn't we say that they were in _covariant_ position as per the earlier argument?
+
+Yes! That's true as well. **The types of `var` fields are in _covariant AND contravariant_ position**. The only way they would work is if the generic type were invariant, which eliminates the problem (same type argument everywhere, no need for substitution).
+
+### 3.3. Types of Method Arguments Are in Contravariant Position
+
+This says it all. How do we prove it? We try the reverse and see how it's wrong.
+
+Let's take the (now) classical example of a list. A list is covariant, we know that. So what would be wrong with
+
+```scala
+abstract class MyList[+T] {
+ def head: T
+ def tail: MyList[T]
+ def add(elem: T): MyList[T]
+}
+```
+
+Let's imagine this code compiled. Let's also imagine we gave some dummy implementations to these methods, as the implementation doesn't matter, only their signature. As per the covariance declaration, we could say
+
+```scala
+val animals: MyList[Animal] = new MyList[Cat]
+val moreAnimals = animals.add(new Dog)
+```
+
+which again breaks the type guarantees. Line 2 allows us to add an Animal to `animals`, and a Dog is an Animal, so we can. However, at line 1, we are using a `MyList[Cat]`, which bears the guarantee that the list contains just cats, and we've just added a Dog to it, which breaks the type checker. Therefore, we say that **the type of method arguments is in _contravariant_ position**.
+
+The (contravariant) Vet example works:
+
+```scala
+class Vet[-T] {
+ def heal(animal: T): Boolean = true // implementation unimportant
+}
+
+val lassiesVet: Vet[Dog] = new Vet[Animal]
+lassiesVet.heal(lassie) // correct; it's a Dog, which is also an Animal; Vet[Animal] can heal any Animal
+lassiesVet.heal(garfield) // legitimate error: Lassie's vet heals Dogs, so Cats aren't allowed
+```
+
+### 3.4. Method Return Types Are in Covariant Position
+
+Again, we can prove the title by trying the reverse. Assume a contravariant type (again, our favorite Vet) and write a method returning a T:
+
+```scala
+abstract class Vet[-T] {
+ def rescueAnimal(): T
+}
+```
+
+In this case, we can write
+
+```scala
+val vet: Vet[Animal] = new Vet[Animal] {
+ override def rescueAnimal(): Animal = new Cat
+}
+
+val lassiesVet: Vet[Dog] = vet // OK because it's a Vet[Animal]
+val rescuedDog: Dog = vet.rescueAnimal // what's up dawg, I'm a Cat
+```
+
+Again, breaking the type guarantees: we use a `Vet[Animal]` whose method returns a `Cat`, so when we finally invoke the method on `lassiesVet` (which is declared as `Vet[Dog]`), the type checker expects a `Dog` but we get a `Cat` per its real implementation! Not funny.
+
+Therefore, we say **method return types are in _covariant_ position**. A covariant example works fine for this case.
+
+## 4. How to Solve the Variance Positions Problem
+
+We proved that a covariant list cannot have an `add(elem: T)` method because it breaks type guarantees. However, does that forbid us from ever adding an element to a list?!
+
+Hell, no.
+
+Let's take this back to first principles. We [said](#33-types-of-method-arguments-are-in-contravariant-position) that we can't add an `add(elem: T)` method to a list, because otherwise we could write
+
+```scala
+val animals: MyList[Animal] = new MyList[Dog]
+val moreAnimals = animals.add(new Cat)
+```
+
+Go back to real life: if we had 3 dogs and a cat, what would the group of 4 be called?
+
+Animals. The most specific type that describes all four.
+
+What if our `add(elem: T): MyList[T]` received an argument of a different type and returned a result of a different type? Allow me to make a suggestion:
+
+```scala
+def add[S >: T](elem: S): MyList[S]
+```
+
+In other words, if we happen to add an element of a different type than the original list, we'll let the compiler infer the lowest type S which describes both the element being added AND the existing elements of the list. As a result, we'll obtain a `MyList[S]`. In our cats vs dogs example, if we add a cat to a list of dogs, we'll obtain a list of animals. If we add a flower to it, we'll obtain a list of life forms. If we add a number, we'll obtain a list of Any. You get the idea.
+
+This is how we solve the cryptic "covariant type T occurs in contravariant position":
+
+```scala
+abstract class MyList[+T] {
+ def add[S >: T](elem: S): MyList[S]
+}
+```
+
+Similarly, we can solve the opposite "contravariant type occurs in covariant position" with the opposite type bound:
+
+```scala
+abstract class Vet[-T] {
+ def rescueAnimal[S <: T](): S
+}
+```
+
+Assuming we can actually implement this method in such general terms, the compiler would be happy: we can force the Vet to return an instance of a particular type:
+
+```scala
+val lassiesVet: Vet[Dog] = new Vet[Animal]
+val rescuedDog: Dog = vet.rescueAnimal[Dog] // type checking passes now
+```
+
+## 5. Conclusion
+
+This is one of the hardest parts of the Scala type system, even though it starts from such an innocent question (should lists of dogs be lists of animals). If you're curious, the [advanced Scala course](https://rockthejvm.com/p/advanced-scala) contains what you've just learned (and lots more on Scala's type system) with some practice exercises.
+
+This article was pretty dense, and we learned quite a bit:
+
+- the variance question: if A extends B, should `Thing[A]` be a subtype of `Thing[B]`?
+- variance possibilities as answers to the variance question: covariant (yes), invariant (no), contravariant (hell no, backwards)
+- types `val` fields are in covariant position
+- types of `var` fields are in covariant AND contravariant position
+- types of method arguments are in contravariant position
+- method return types are in covariant position
+- we solve the "covariant type occurs in contravariant position" by "widening": we add a type argument `[S >: T]` and change the argument type to S
+- we solve the "contravariant type occurs in covariant position" by "narrowing": we add a type argument `[S <: T]` and change the method return type to S
+
+## 6. Watch on YouTube
+
+{% include video id="aUmj7jnXet4" provider="youtube" %}
diff --git a/_posts/2020-11-16-scala-3-given-using.md b/_posts/2020-11-16-scala-3-given-using.md
new file mode 100644
index 000000000000..b4c5c11ff8df
--- /dev/null
+++ b/_posts/2020-11-16-scala-3-given-using.md
@@ -0,0 +1,194 @@
+---
+title: "Given and Using Clauses in Scala 3"
+date: 2020-11-16
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala 3]
+excerpt: The given/using clauses are some of the most important features of Scala 3. Read on for the essential concepts and how to use them."
+---
+
+In this article, we'll take a structured look at the new given/using clauses in Scala 3, which promises to be a big leap forward.
+
+Unlike other posts on the topic, this article is also approachable for Scala beginners. The given/using pair in Scala 3 is often described in comparison with implicits — which are themselves really powerful and hard to learn if you're starting out — but here I'll make no such references or assumptions.
+
+So if you're starting out directly on Scala 3, or you're a Scala 2 developer without too much experience with implicits, this one is for you. If you happen to know how implicits work, that'll only be a plus.
+
+This feature (along with dozens of other changes) is explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## A Small Problem
+
+Here's a situation. Let's say we write a nation-wide census application. Something like the following case class (perhaps with some more fields) would be used everywhere:
+
+```scala3
+case class Person(surname: String, name: String, age: Int)
+```
+
+It would make sense for instances of Person to be ordered in various data structures. For applications like these, ordering Persons needs to have a "standard" algorithm, usually alphabetically by surname:
+
+```scala3
+val personOrdering: Ordering[Person] = new Ordering[Person] {
+ override def compare(x: Person, y: Person): Int =
+ x.surname.compareTo(y.surname)
+}
+```
+(you can use your favorite comparison class instead of `Ordering` if you like)
+
+In other words, we need a single standard `Ordering[Person]` that we need to use everywhere. At the same time, making that instance available and using it explicitly would make the code cumbersome, because such a standard ordering is _assumed_, and (as a developer) we want to focus on the logic rather than pass the same standard value everywhere:
+
+```scala3
+def listPeople(persons: Seq[Person])(ordering: Ordering[Person]) = ...
+def someOtherMethodRequiringOrdering(alice: Person, bob: Person)(ordering: Ordering[Person]) = ...
+def yetAnotherMethodRequiringOrdering(persons: List[Person])(ordering: Ordering[Person]) = ...
+```
+
+When we call these methods, we first need to
+
+- find the standard ordering, without needing to re-instantiate it
+- plug it into all these methods
+
+Instead of doing this explicitly every single time for every single method, we can delegate this menial task to the compiler.
+
+## Given/Using Clauses
+
+We'll change our code in two small ways.
+
+First, our standard ordering will be considered a "given", that is, an automatically created instance which is readily available to be injected in the "right place". The structure of the declaration will look like this:
+
+```scala3
+given personOrdering: Ordering[Person] with {
+ override def compare(x: Person, y: Person): Int =
+ x.surname.compareTo(y.surname)
+}
+```
+
+So notice we aren't "assigning" the Ordering instance to a value. We are marking that instance as a "standard" (or a "given"), and we attach a name to it. The name is useful so we can reference this instance and use fields and methods on it, much like any other value.
+
+However, we also need to mark the places where this given should automatically be injected. For this, we mark the relevant method argument with the `using` clause:
+
+```scala3
+def listPeople(persons: Seq[Person])(using ordering: Ordering[Person]) = ...
+```
+
+So that when we invoke this method, we don't need to explicitly pass the `ordering` argument:
+
+```scala3
+listPeople(List(Person("Weasley", "Ron", 15), Person("Potter", "Harry", 15))) // <- the compiler will inject the ordering here
+```
+
+This leads to much cleaner code, especially in large function call chains.
+
+## Importing Givens
+
+The compiler can inject a given instance where a `using` clause is, if it has access to a given instance of that type in scope. If we continue the example above, a large-scale application will not be written in a single file, so we need a mechanism for importing given instances.
+
+Let's assume our given instance stays in an `object`:
+
+```scala3
+object StandardValues {
+ given personOrdering: Ordering[Person] with {
+ override def compare(x: Person, y: Person): Int = x.surname.compareTo(y.surname)
+ }
+}
+```
+
+We would import the given instance as
+
+```scala3
+import StandardValues.personOrdering
+```
+
+which would make it explicit and easy to track down. Alternatively, if we wanted to import a given instance of a particular type — there can only be one — we could say:
+
+```scala3
+import StandardValues.{given Ordering[Person]}
+```
+
+Note that the regular wildcard import `import StandardValues._` would import all definitions _except_ given instances. If we want to bring all given values in scope, we could write:
+
+```scala3
+import StandardValues.{given _}
+```
+
+## Deriving Givens
+
+As we saw earlier, a given instance will be automatically created and injected where a `using` clause is present. Taking this concept further, what if we had a given instance that depends on another given instance, via a `using` clause?
+
+In Scala 3, we can.
+
+Let's imagine that in our big census application we have many types for which we have `given` instances of `Ordering`. Meanwhile, because we're using pure FP to deal with value absence, we're working with Options, and we need to compare them, sort them etc. Can we automatically create an `Ordering[Option[T]]` if we had an `Ordering[T]` in scope?
+
+```scala3
+given optionOrdering[T](using normalOrdering: Ordering[T]): Ordering[Option[T]] with {
+ def compare(optionA: Option[T], optionB: Option[T]): Int = (a, b) match {
+ case (None, None) => 0
+ case (None, _) => -1
+ case (_, None) => 1
+ case (Some(a), Some(b)) => normalOrdering.compare(a, b)
+ }
+}
+```
+
+This structure tells the compiler, "if you have a given instance of `Ordering[T]` in scope, then you can automatically create a new instance of `Ordering[Option[T]]` with the implementation following the brace. Behind the scenes, the new given structure works similar to a method. If we ever need to call a method such as
+
+```scala3
+def sortThings[T](things: List[T])(using ordering: Ordering[T]) = ...
+
+// elsewhere in our code
+val maybePersons: List[Option[Person]] = ...
+sortThings(maybePersons)
+```
+
+the compiler will automatically create an `Ordering[Option[Person]]` based on the existing `Ordering[Person]`, so the call will look like
+
+```scala3
+sortThings(maybePersons)(optionOrdering(personOrdering))
+```
+
+Of course, that's not what we see (because we don't see anything), but this serves as an analogy to better understand the processes under the hood.
+
+## Where Givens Are Useful
+
+The problem we started with was pretty small, but it's also the easiest to lean into. Given/using clauses, in combination with extension methods — coming in another article — are a powerful cocktail of tools, which can be used for (among others):
+
+- type classes
+- dependency injection
+- contextual abstractions, i.e. ability to use code for some types but not for others
+- automatic type creation
+- type-level programming
+
+We will explore lots of these problems and how given/using clauses + extension methods solve them as the blog evolves.
+
+In the simplest terms, a `using` clause is a marker to the compiler, so that if it can find a `given` instance of that type in the scope where that definition is used (e.g. a method call), the compiler will simply take that given instance and inject it there.
+
+The obvious restriction is that **there cannot be two `given` instances of the same type in the same scope**, otherwise the compiler would not know which one to pick.
+
+More philosophically, a `given` proves the existence of a type. If the existence of a type can be proven by the compiler, new given instances can be constructed, if they rely on a `using` clause. If we combine given/using combos for certain types, we can prove type relationships at compile time, in a style that looks like [this](/type-level-programming-part-1). In a future article, I'll show you how we can run type-level computations with givens in Scala 3.
+
+## Other Niceties
+
+Notice that in the previous example with the person ordering, once we used the given/using combo, we didn't even need the name of the given instance. In that case, we can simply write:
+
+```scala3
+given Ordering[Person] {
+ override def compare(x: Person, y: Person): Int =
+ x.surname.compareTo(y.surname)
+}
+```
+
+Sometimes defining instances on the spot might not be convenient, when we already have simpler/better construction tools available (e.g. factory methods, existing values, better constructors). If that is the case, we can create a given instance where the value of it is an expression:
+
+```scala3
+given personOrdering: Ordering[Person] = Ordering.fromLessThan((a, b) => a.surname.compareTo(b.surname) < 0)
+```
+
+or even make it anonymous:
+
+```scala3
+given Ordering[Person] = Ordering.fromLessThan((a, b) => a.surname.compareTo(b.surname) < 0)
+```
+
+## Conclusion
+
+The `given` structure allows an instance of a certain type to be automatically constructed, available and inserted wherever a `using` clause for that type is present.
+
+That sentence took me 15 minutes to write.
diff --git a/_posts/2020-11-18-givens-vs-implicits.md b/_posts/2020-11-18-givens-vs-implicits.md
new file mode 100644
index 000000000000..9f3330eabead
--- /dev/null
+++ b/_posts/2020-11-18-givens-vs-implicits.md
@@ -0,0 +1,152 @@
+---
+title: "Givens vs. Implicits in Scala 3"
+date: 2020-11-18
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala 3]
+excerpt: From the previous article, you know how givens work. Let's compare them with the old Scala implicits.
+---
+
+This article is for the Scala programmers who have some familiarity with implicits.
+
+If you're just starting out and got to Scala 3 directly, [the essential concepts](/scala-3-given-using) of given/using will be enough. No need to read this article because implicits are phased out in Scala 3. To use some word play here, just stick to `using` `given`s.
+
+If you come from Scala 2, you're familiar with implicits and need to move to given/using combos, read on.
+
+## Background: Overhaul of Implicits
+
+Implicits are some of the most powerful Scala features. They allow code abstractions that sit outside the traditional OO-style type hierarchy. Implicits have demonstrated their use and have been battle-tested in many scenarios. Listing just a few:
+
+- Implicits are an essential tool for creating [type classes](/why-are-typeclasses-useful/) in Scala. Libraries like [Cats](https://typelevel.org/cats) (which we [teach](https://rockthejvm.com/p/cats)) could not exist without them.
+- Extending the capabilities of existing types, in expressions such as `20.seconds` (more [here](/twenty-seconds)), is possible with implicits.
+- Implicits allow the automatic creation of new types and enforcing type relationships between them at compile time. We can go as far as run [type-level computations in Scala](/type-level-programming-part-1) with implicits.
+
+In Scala 2, this suite of capabilities is available under the same `implicit` keyword, through implicit `val`s, implicit `def`s and implicit classes. However, this unified scheme has its downsides, and implicits have garnered criticism. Some of the most important:
+
+1. When we have some working code using implicits, it's often very hard — and exponentially harder with a growing codebase — to pinpoint which implicits made it possible.
+2. When we write code that requires implicits, we often need to import the _right_ implicits to make it compile. Automatic imports are really hard — the IDE can't read your mind — so that leaves us with either a) magically knowing which imports to pick, or b) frustration.
+3. Implicit `def`s without implicit arguments are capable of doing conversions. Most of the time, these conversions are dangerous and hard to pin down. Moreover, they're the easiest implicits feature to use, which makes them double-dangerous.
+4. Implicits are really hard to learn and therefore push many beginners away from Scala.
+4. Various annoyances, such as
+ - the need to name implicits when we often don't need them
+ - some syntax confusions if a method requires implicit parameters
+ - the discrepancy between structure and intention: for example, an `implicit def` is never used with the meaning of a "method".
+
+## Implicit Conversions
+
+Implicit conversions now need to be made explicit. This solves a big burden.
+
+Prior to Scala 3, implicit conversions were required for extension methods and for the type class pattern. Now, with Scala 3, the extension method concept is standalone, and so we can implement many of the patterns that required implicits without relying on conversions. As such, it's quite likely that the need for conversions will drop significantly. At the same time, conversions are dangerous on their own, and because they used to be so sneaky, they're double-dangerous. With Scala 3, conversions need to be declared in a specific way.
+
+Prior to Scala 3, implicit conversions were incredibly easy to write compared to their power (and danger). Assuming a class
+
+```scala3
+case class Person(name: String) {
+ def greet: String = s"Hey, I'm $name. Scala rocks!"
+}
+```
+
+we could write a one-liner implicit conversion as
+
+```scala3
+implicit def stringToPerson(string: String): Person = Person(string)
+```
+
+and then we could write
+
+```scala3
+"Alice".greet
+```
+
+Now, with Scala 3, there are many steps to follow to make sure we know what we're doing. An implicit conversion is a `given` instance of `Conversion[A, B]`. The example of the Person class would be
+
+```scala3
+given stringToPerson: Conversion[String, Person] with {
+ def apply(s: String): Person = Person(s)
+}
+```
+
+but we still wouldn't be able to rely on the implicit magic. We also need to specifically import the `implicitConversions` package. So we need to write
+
+```scala3
+import scala.language.implicitConversions
+
+// somewhere in the code
+"Alice".greet
+```
+
+In this way, you need to be really motivated to use implicit conversions. Coupled with the lack of proper reasons to use implicit conversions — we don't need them for extension methods anymore — should make the use of implicit conversions drop dramatically.
+
+## Scala 3 Givens, Implicits and Naming
+
+Firstly, Scala 2 implicits needed to be named, even though we might never need to refer to them. Not anymore with givens. You can simply a given instance without naming it:
+
+```scala3
+given Ordering[String] {
+ // implementation
+}
+```
+
+and at the same time, write `using` clauses without naming the value which will be injected:
+
+```scala3
+def sortThings[T](things: List[T])(using Ordering[T]) = ...
+```
+
+## Givens, Implicits and Syntax Ambiguities
+
+Secondly, givens solve a syntax ambiguity when invoking methods which have `using` clauses. Let's take an example. If we had a method
+
+```scala3
+def getMap(implicit size: Int): Map[String, Int] = ...
+```
+
+then we could not write `getMap("Alice")` even if we had an implicit in scope, because the argument will override the implicit value the compiler would have inserted, and so we'll get a type error from the compiler.
+
+Givens solve that. If we had a method
+
+```scala3
+def getMap(using size: Int): Map[String, Int] = ...
+```
+
+we cannot call the method explicitly with an argument of our choosing to be passed for `size`, unless we are also explicit about it:
+
+```scala3
+getMap(using 42)("Alice")
+```
+
+which again is very clear. If we do have a `given` Int in scope, then we can simply call `getMap("Alice")`, because the given value was already injected into `size`.
+
+## How Scala 3 Givens Solve the Track-Down Problem
+
+Implicits are notorious in Scala 2 for being extremely hard to pin down. That means that in a large chunk of working code, you may be using methods that take implicit arguments, be using implicit conversions and/or methods that don't belong to the type you're using (extension methods), _and still have no idea where they come from_.
+
+Givens attempt at solving the problem in multiple ways.
+
+Firstly, given instances need to be [explicitly imported](/scala-3-given-using/#importing-givens), so you can better track down which imported parts are actually given instances.
+
+Secondly, givens are only used for automatic injection of arguments via a `using` clause. In this way, you can look at imported given instances for this particular issue, i.e. finding _method arguments that you aren't passing explicitly_. For the other implicit magic, the other mechanisms (clearly defined implicit conversions and extension methods) have similar track-down capabilities.
+
+## Scala 3 Givens and Auto-Imports
+
+This is a hard one. Because givens are automatically injected wherever a `using` clause for that type is present, this mechanism is similar to implicits. If we call a method which has a `using` clause:
+
+```scala3
+def sortTheList[T](list: List[T])(using comparator: Comparator[T]) = ...
+```
+
+then the IDE cannot read our mind and automatically import the right `given` instance in scope so we can call our method. Imports will still need to be explicit.
+
+However, the current implicit resolution mechanism leaves very generic errors. At most, "no implicits found". The Scala 3 compiler has come a long way to surface more meaningful errors, so that if the search for `given`s fails, it will show the point where the compiler got stuck, so we can provide the remaining `given` instances in scope.
+
+## Scala 3 Givens, Implicits and Intentions
+
+Implicit defs were never meant to be used like methods. Therefore, there's a clear discrepancy between the structure of the code (a method) and the intention (a conversion). The new Scala 3 contextual abstractions solve this problem by being very clear on the intent:
+
+- given/using clauses are used for passing "implicit" arguments
+- implicit conversions are done by creating instances of `Conversion`
+- extension methods have their first-class syntactic structure
+
+## Conclusion
+
+Implicits are powerful, dangerous and one of the features that make Scala unique. Scala 3 moves beyond the implicit mechanism with much clearer intention of which feature wants to achieve what. The new world with given/using + extension methods + explicit implicit conversions will encounter some push-back because of current familiarity, but looking a few years into the future, I'm optimistic we'll look back to now and be glad we write clearer Scala code because of this new move.
diff --git a/_posts/2020-11-27-scala-3-dependent-types.md b/_posts/2020-11-27-scala-3-dependent-types.md
new file mode 100644
index 000000000000..cc0b7dd51851
--- /dev/null
+++ b/_posts/2020-11-27-scala-3-dependent-types.md
@@ -0,0 +1,149 @@
+---
+title: "Scala 3: Path-Dependent Types, Type Projections, Dependent Methods and Functions"
+date: 2020-11-27
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3, type system]
+excerpt: "This quick tutorial will show you what dependent types are and how they work in Scala 3, along with dependent methods and functions."
+---
+
+This short article is for the Scala developer who is curious about the capabilities of its type system. What I'm about to describe is not used very often, but when you need something like this, it can prove pretty powerful.
+
+If you want to understand why abstract type projections are unsound and were removed in Scala 3, check [this article](/scala-3-type-projections/).
+
+## 1. Nesting Types
+
+You're probably well aware that classes, objects and traits can hold other classes, objects and traits, as well as define type members — abstract or concrete in the form of type aliases.
+
+```scala3
+class Outer {
+ class Inner
+ object InnerObj
+ type InnerType
+}
+```
+
+The question of using those types from inside the `Outer` class is easy: all you have to do is just use those nested classes, objects or type aliases by their name.
+
+The question of using those types from _outside_ the `Outer` class is a bit trickier. For example, we would only be able to instantiate the `Inner` class if we had access to an instance of `Outer`:
+
+```scala3
+val outer = new Outer
+val inner = new o.Inner
+```
+
+The type of `inner` is `o.Inner`. In other words, each instance of `Outer` has its own nested types! For example, it would be a type mismatch if we wrote:
+
+```scala3
+val outerA = new Outer
+val outerB = new Outer
+val inner: outerA.Inner = new outerB.Inner
+```
+
+The same thing goes for the nested singleton objects. For example, the expression
+
+```scala3
+outerA.InnerObj == outerB.InnerObj
+```
+
+would return false. Similarly, the abstract type member `InnerType` is different for every instance of `Outer`.
+
+## 2. Path-Dependent Types and Type Projections
+
+Let's assume we had a method in the class `Outer`, of the form
+
+```scala3
+class Outer {
+ // type definitions
+ def process(inner: Inner): Unit = ??? // give a dummy implementation, like printing the argument
+}
+```
+
+With this kind of method, we can only pass `Inner` instances that correspond to the `Outer` instance that created them. Example:
+
+```scala3
+val outerA = new Outer
+val innerA = new outerA.Inner
+val outerB = new Outer
+val innerB = new outerB.Inner
+
+outerA.process(innerA) // ok
+outerA.process(innerB) // error: type mismatch
+```
+
+This is expected, since `innerA` and `innerB` have different types. However, there is a parent type for all `Inner` types: `Outer#Inner`.
+
+```scala3
+class Outer {
+ // type definitions
+ def processGeneral(inner: Outer#Inner): Unit = ??? // give a dummy implementation, like printing the argument
+}
+```
+
+The type `Outer#Inner` is called a _type projection_. With this definition, we can now use `Inner` instances created by any `Outer` instance:
+
+```scala3
+outerA.processGeneral(innerA) // ok
+outerA.processGeneral(innerB) // ok
+```
+
+The types of the style `instance.MyType` and `Outer#Inner` are called _path-dependent types_, because they depend on either an instance or an outer type (a "path"). The term is quite confusing — as some feedback to this article and the [video](https://www.youtube.com/watch?v=63syJfNoDPI) has pointed out — and I'll use the term _type projection_ to differentiate the `Outer#Inner` types from the rest.
+
+## 3. Motivation for Path-Dependent Types and Type Projections
+
+Here are a few examples where path-dependent types type projections are useful.
+
+Example 1: a number of libraries use type projections for type-checking and type inference. [Akka Streams](https://doc.akka.io/docs/akka/current/stream/index.html), for example, uses path-dependent types to automatically determine the appropriate stream type when you plug components together: for example, you might see things like `Flow[Int, Int, NotUsed]#Repr` in the type inferrer.
+
+Example 2: [type lambdas](/scala-3-type-lambdas/) used to rely exclusively on type projections in Scala 2, and they looked pretty hideous (e.g. `{ type T[A] = List[A] }#T` ) because it was essentially the only way to do it. Thank heavens we now have a proper syntactic construct in Scala 3 for type lambdas.
+
+Example 3: you might even go bananas and write a full-blown [type-level sorter](/type-level-programming-part-1/) by abusing abstract types and instance-dependent types along with implicits (or givens in Scala 3).
+
+## 4. Methods with Dependent Types
+
+Now, with this background in place, we can now explore methods that rely on the type of the argument to return the appropriate nested type. For example, if you had a data structure/record description for some data access API:
+
+```scala3
+class AbstractRow {
+ type Key
+}
+```
+
+the following method would compile just fine:
+
+```scala3
+def getIdentifier(row: AbstractRow): row.Key = ???
+```
+
+Besides generics, this is the only technique I know that would allow a method to return a different type depending on the value of the argument(s). This can prove really powerful in libraries.
+
+## 5. Functions with Dependent Types
+
+This is new in Scala 3. Prior to Scala 3, it wasn't possible for us to turn methods like `getIdentifier` into function values so that we can use them in higher-order functions (e.g. pass them as arguments, return them as results etc). Now we can, by the introduction of the _dependent function types_ in Scala 3. Now we can assign the method to a function value:
+
+```scala3
+val getIdentifierFunc = getIdentifier
+```
+
+and the type of the function value is `(r: AbstractRow) => r.Key`.
+
+To bring this topic full-circle, the new type `(r: AbstractRow) => r.Key` is syntax sugar for
+
+```scala3
+Function1[AbstractRow, AbstractRow#Key] {
+ def apply(arg: AbstractRow): arg.Key
+}
+```
+
+which is a subtype of `Function1[AbstractRow, AbstractRow#Key]` because the `apply` method returns the type `arg.Key`, which we now [know](#2-path-dependent-types-and-type-projections) that it's a subtype of `AbstractRow#Key`, so the override is valid.
+
+## 6. Conclusion
+
+Let's recap:
+
+ - we covered nested types and the need to create different types for different outer class instances
+ - we explored type projections, the mother of instance-dependent types (`Outer#Inner`)
+ - we went through some examples why path-dependent types and type projections are useful
+ - we discussed dependent methods and dependent _functions_, the latter of which is exclusive to Scala 3
+
+Hope it helps!
diff --git a/_posts/2020-12-04-immutable-doubly-linked-list-scala.md b/_posts/2020-12-04-immutable-doubly-linked-list-scala.md
new file mode 100644
index 000000000000..4fe951488b97
--- /dev/null
+++ b/_posts/2020-12-04-immutable-doubly-linked-list-scala.md
@@ -0,0 +1,283 @@
+---
+title: "Immutable Doubly Linked Lists in Scala with Call-By-Name and Lazy Values"
+date: 2020-12-04
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, data structures]
+excerpt: "I'll show you a technique to use lazy values and call-by-name for recursive value definitions and implement a fully immutable doubly-linked list in Scala."
+---
+
+This article is for the Scala programmers who want to brush up on their data structures. However, even seasoned developers might find the following problem challenging:
+
+> How would you write a simple, fully immutable, doubly-linked list in Scala?
+
+If you're up for a challenge, try your hand at this problem before reading the rest of the article. You might find it's not quite what you'd expect.
+
+This article was inspired by a question from one of my students who got the point of my initial singly linked list implementation and wanted to move one level up. However simple a doubly-linked-list might be conceptually, this student picked quite a difficult data structure for practice.
+
+## 1. The Disclaimer
+
+A doubly-linked list doesn't offer too much benefit in Scala, quite the contrary. The complexity is still O(1) for accessing the "head" i.e. the current element you're pointing at, and O(n) for basically everything else. While appending was O(n) for singly-linked lists, now you'll get O(n) for _prepending_ as well for doubly-linked lists. Not to mention you'd have to replace the entire list for every modification.
+
+So this data structure is pretty useless from a practical standpoint, which is why (to my knowledge) there is no doubly-linked list in the Scala standard collection library. The only utility in having a doubly-linked list is the ability to traverse the list in both directions.
+
+All that said, a doubly-linked list is an excellent exercise on the mental model of immutable data structures.
+
+## 2. The Challenge with a Doubly-Linked List
+
+Implementing a _singly_ linked list is relatively easy. Define a list interface with some basic methods:
+
+```scala
+trait MyList[+T] {
+ def head: T
+ def tail: MyList[T]
+ def ::[S >: T](element: S): MyList[S]
+}
+```
+
+with a type widening `[S >: T]` to save ourselves the trouble of the [cryptic variance positions error](/scala-variance-positions/) (read that article first to understand what's happening, if you're curious about it).
+
+Then the implementation would be pretty straightforward: an empty object which is a `MyList[Nothing]`, and a non-empty `Cons[+T]` — Cons stands for "constructor", an ancient primitive for lists in functional languages. So you'd end up with something like this:
+
+```scala
+case object Empty extends MyList[Nothing] {
+ override def head: Nothing = throw new NoSuchElementException("head of an empty list")
+ override def tail: Nothing = throw new NoSuchElementException("tail of an empty list")
+ override def ::[S >: Nothing](element: S): MyList[S] = Cons(element, Empty)
+}
+
+// head/tail are vals, they don't need to be evaluated every time
+case class Cons[+T](override val head: T, override val tail: MyList[T]) extends MyList[T] {
+ override def ::[S >: T](element: S): MyList[S] = Cons(element, this)
+}
+```
+
+The point being that adding an element is straightforward: create a new node, point the tail to the existing list, then return the new node as the head of the resulting list.
+
+Let's try the same with a doubly-linked list. This time we have two list references (because we can traverse both ways), and we have two addition methods because we can add to either end of the list:
+
+```scala
+trait DLList[+T] {
+ def value: T // "head" doesn't feel right
+ def prev: DLList[T]
+ def next: DLList[T]
+ def prepend[S >: T](element: S): DLList[S]
+ def append[S >: T](element: S): DLList[S]
+}
+```
+
+Assuming we had a similar Cons-like data type for a non-empty list, the Empty case would be straightforward:
+
+```scala
+case object DLEmpty extends DLList[Nothing] {
+ override def value = throw new NoSuchElementException("head of an empty list")
+ override def prev = throw new NoSuchElementException("prev of an empty list")
+ override def next = throw new NoSuchElementException("tail of an empty list")
+
+ override def prepend[S >: Nothing](element: S) = new DLCons(element, DLEmpty, DLEmpty)
+ override def append[S >: Nothing](element: S) = new DLCons(element, DLEmpty, DLEmpty)
+}
+```
+
+Now the Cons thing (which I've named DLCons) is not straightforward. Let's say we're looking at the list `[1,2,3]` and our "pointer" is at 2. How do we prepend an element to this list? We'd have to find the starting node of the list, add a new one, and have its `next` reference point to the existing list, correct?
+
+Not so fast.
+
+ - Because this is a doubly-linked list, the existing "left-end" of the list would have to point to the new node as well.
+ - Which means we'd have to replace it with a new node.
+ - However, this new node's `next` reference would have to point to the remainder of the list.
+ - Which means that we'd have to replace this existing node in the list with a new node as well.
+ - And so on, for the entire list...
+ - ...without any mutable variables.
+
+Turns out that's hard to do in a single operation. We can't have two declared nodes reference each other, because at least one would have to be a forward declaration. Demo with a circular singly linked list (which doesn't work):
+
+```scala
+val a = Cons(3, b) // who's b? I can't create a
+val b = Cons(4, a) // who's a?
+```
+
+An imperative approach would quickly solve this problem because we're allowed to mutate references: for example we could start with nodes pointing at `null`, then change the references to point to the right nodes. But we're not allowed mutation, so we have our hands tied.
+
+## 3. Enter Call-By-Name and Lazy Vals
+
+This section assumes you know call-by-name and lazy values. A quick recap:
+
+ - The marker `=>` attached to a method argument means that this argument will _not_ be evaluated until needed, and it will be evaluated as many times as it's used in the method body. This is a "call-by-name", or simply by-name, argument.
+ - A `lazy` variable (either `val` or `var`) means that the variable will only be evaluated when used _for the first time_. After the first use, the value will be already set and won't need recomputing.
+
+These two features have pretty powerful consequences, especially [when used in conjunction](/3-tricks-for-cbn/#trick-2---manageable-infinity).
+
+For our use-case this delayed computation allows us to use _forward references_. Here is a first stab at the non-empty doubly-linked list:
+
+```scala
+class DLCons[+T](override val value: T, p: => DLList[T], n: => DLList[T]) extends DLList[T] {
+ override lazy val prev: DLList[T] = p
+ override lazy val next: DLList[T] = n
+}
+```
+
+Obviously, the `prepend` and `append` methods are the meat of the problem. Let's take prepending, without loss of generality — appending will be implemented in a perfectly symmetrical way.
+
+As we mentioned earlier, prepending means:
+
+ - finding the "left-end" of the list
+ - adding a new node
+ - pointing that node's `next` reference to the rest of the list...
+ - ...which also needs to update its `prev` and `next` reference throughout the list
+
+## 4. Forward References and Lazy Vals
+
+Let's take the smaller problem of updating a node's `prev` reference. Let's say we have the list `[1,2,3,4]`, pointing at 1. We want to change this node's `left` reference to another value, say 5, so we'll end up with `[5,1,2,3,4]`.
+
+First, we need to create another node whose `prev` reference now points where we wanted.
+
+```
+old: 1 ⇆ 2 ⇆ 3 ⇆ 4
+new: 5 ← 1
+```
+
+We then need to update the `prev` reference of the _next_ node (this case 2), while updating the `next` reference of the new 1, _in the same operation_.
+
+```
+old: 1 ⇆ 2 ⇆ 3 ⇆ 4
+new: 5 ← 1 ⇆ 2
+```
+
+Think about it: this operation in Java would have been done in 3 steps:
+
+ - new node
+ - new node's prev is 1
+ - 1's next is the new node
+
+If we can execute that new `1 ⇆ 2` _in the same expression_, we've made it; we can then recursively apply this operation on the rest of the list. Strangely enough, it's possible. Here's how we can do it. We'll define two more methods in the main trait:
+
+```scala
+trait DLList[+T] {
+ // ... other methods
+ def updatePrev[S >: T](newPrev: => DLList[S]): DLList[S]
+ def updateNext[S >: T](newNext: => DLList[S]): DLList[S]
+}
+```
+
+(notice the by-name argument)
+These methods will be straightforward in the Empty case since there's no reference to update:
+
+```scala
+case object DLEmpty extends DLList[Nothing] {
+ // ... other methods
+ override def updatePrev[S >: Nothing](newPrev: => DLList[S]): DLList[S] = this
+ override def updateNext[S >: Nothing](newNext: => DLList[S]): DLList[S] = this
+}
+```
+
+Now for the non-empty list, we'll use lazy vals and we'll exploit the fact that the constructor arguments for `DLCons` are by-name:
+
+```scala
+class DLCons[+T](override val value: T, p: => DLList[T], n: => DLList[T]) extends DLList[T] {
+ // ... other methods
+
+ override def updatePrev[S >: T](newPrev: => DLList[S]) = {
+ lazy val result: DLCons[S] = new DLCons(value, newPrev, n.updatePrev(result))
+ result
+ }
+
+ override def updateNext[S >: T](newTail: => DLList[S]) = {
+ lazy val result: DLCons[S] = new DLCons(value, p.updateNext(result), newTail)
+ result
+ }
+}
+```
+
+Look at `updatePrev`: we're creating a new node, whose `next` reference immediately points to a recursive call; the recursive call is on the current `next` node, which will update its own previous reference to the result we're still in the process of defining! This forward reference is only possible in the presence of lazy values and the by-name arguments. The `updateNext` method is simply symmetrical.
+
+## 5. Adding Elements to a Doubly-Linked list
+
+With these methods in place, we can now use a similar technique to add another element to either ends of a doubly-linked list:
+
+```scala
+class DLCons[+T](override val value: T, p: => DLList[T], n: => DLList[T]) extends DLList[T] {
+ // ... other methods
+ def append[S >: T](element: S): DLList[S] = {
+ lazy val result: DLList[S] = new DLCons(value, p.updateNext(result), n.append(element).updatePrev(result))
+ result
+ }
+
+ def prepend[S >:T](element: S): DLList[S] = {
+ lazy val result: DLList[S] = new DLCons(value, p.prepend(element).updateNext(result), n.updatePrev(result))
+ result
+ }
+}
+```
+
+Look at the `prepend` method: we're creating a new node with the same value as the one we're looking at, whose previous node is a recursive call to `prepend` _with its `next` pointer updated to the node we're currently defining_ (the same forward reference technique), and the "tail" of the list simply being an update of the current "tail" with the `prev` pointer to the node we're currently defining (yet another forward reference). Again, the `append` method is symmetrical.
+
+## 6. Full Code
+
+```scala
+trait DLList[+T] {
+ def value: T // instead of "head"
+ def prev: DLList[T]
+ def next: DLList[T]
+ def prepend[S >: T](element: S): DLList[S]
+ def append[S >: T](element: S): DLList[S]
+
+ def updatePrev[S >: T](newPrev: => DLList[S]): DLList[S]
+ def updateNext[S >: T](newNext: => DLList[S]): DLList[S]
+}
+
+case object DLEmpty extends DLList[Nothing] {
+ override def value = throw new NoSuchElementException("head of an empty list")
+ override def prev = throw new NoSuchElementException("prev of an empty list")
+ override def next = throw new NoSuchElementException("tail of an empty list")
+
+ override def prepend[S >: Nothing](element: S) = new DLCons(element, DLEmpty, DLEmpty)
+ override def append[S >: Nothing](element: S) = new DLCons(element, DLEmpty, DLEmpty)
+
+ override def updatePrev[S >: Nothing](newPrev: => DLList[S]): DLList[S] = this
+ override def updateNext[S >: Nothing](newNext: => DLList[S]): DLList[S] = this
+}
+
+class DLCons[+T](override val value: T, p: => DLList[T], n: => DLList[T]) extends DLList[T] {
+ override lazy val prev: DLList[T] = p
+ override lazy val next: DLList[T] = n
+
+ override def updatePrev[S >: T](newPrev: => DLList[S]) = {
+ lazy val result: DLCons[S] = new DLCons(value, newPrev, n.updatePrev(result))
+ result
+ }
+
+ override def updateNext[S >: T](newTail: => DLList[S]) = {
+ lazy val result: DLCons[S] = new DLCons(value, p.updateNext(result), newTail)
+ result
+ }
+
+ def append[S >: T](element: S): DLList[S] = {
+ lazy val result: DLList[S] = new DLCons(value, p.updateNext(result), n.append(element).updatePrev(result))
+ result
+ }
+
+ def prepend[S >:T](element: S): DLList[S] = {
+ lazy val result: DLList[S] = new DLCons(value, p.prepend(element).updateNext(result), n.updatePrev(result))
+ result
+ }
+}
+
+// play with the data structure and test it around
+object DLListPlayground {
+ def main(args: Array[String]): Unit = {
+ val list = DLEmpty.prepend(1).append(2).prepend(3).append(4)
+ println(list.value) // 1
+ println(list.next.value) // 2
+ println(list.next.prev == list) // true
+ println(list.prev.value) // 3
+ println(list.prev.next == list) // true
+ println(list.next.next.value) // 4
+ println(list.next.next.prev.prev == list) // true
+ }
+}
+```
+
+## 7. Conclusion
+
+We learned how to use _forward references_ based on call-by-name and lazy values to implement a (pretty primitive) fully immutable doubly-linked list with proper appending and prepending. This data structure will likely not become part of the standard library too soon — we'd have to replace the whole list on every new addition — this is an excellent small exercise for the technique, that you might apply elsewhere in your libraries or custom data structures.
diff --git a/_posts/2020-12-2-the-2020-retrospective.md b/_posts/2020-12-2-the-2020-retrospective.md
new file mode 100644
index 000000000000..ab83cc48ef9b
--- /dev/null
+++ b/_posts/2020-12-2-the-2020-retrospective.md
@@ -0,0 +1,114 @@
+---
+title: "2020 Retrospective (and What's Coming in 2021)"
+date: 2020-12-07
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [meta]
+excerpt: "In this article I'll wrap 2020 and share a few insights, achievements and changes, and I'll project some plans for the future of Rock the JVM."
+---
+
+This year has been interesting to say the least. Almost everyone went (and is still going) through a rollercoaster ride, and I'm no exception. Rock the JVM has undergone massive changes, and I've learned a ton.
+
+In this last article for 2020, I want to share with you the progress Rock the JVM has made so far, as well as my plans for next year.
+
+## 1. The Blog
+
+Around March, I committed to post at least one significant piece every week. After 52 articles, that's one/week for 2020! I'm quite happy with the content I've put out so far and for the reception it's received. Thank YOU!
+
+At the same time, the site that you're looking at is only a few months old! This has been one of the most important changes of the Rock the JVM content stream.
+
+I use Teachable for the [main site](https://rockthejvm.com) and a very attractive feature was the built-in blog, which I set up very quickly and began posting. However, the editing experience is very poor and the rendering is buggy, particularly for code snippets. So after 578293582 workarounds, HTML generators and custom CSS, I was done.
+
+So I learned [Jekyll](https://jekyllrb.com) and set up this site with the [Minimal Mistakes theme](https://mmistakes.github.io) in a few hours, then wrote a migration tool to automatically turn my custom blog post syntax into actual Markdown, with proper article tags. I'm much happier how. I can write Markdown, publish with Git, highlight properly, add photos, add some SEO for more visibility (soon).
+
+Lesson learned: be ready to give up some upfront time for more control later.
+
+## 2. Socials
+
+This has been a complete start-from-scratch effort. Starting from zero followers on every platform feels daunting and intimidating.
+
+I remember it took me a few days to get my first Twitter follower. After following @rockthejvm, this person clicked the like button on every single tweet I sent for a few months. If you're reading this, @blaizedsouza, thank you. The trust of a single person was enough to keep me writing, recording and sharing. As of December 2, I'm looking at 581 followers on [Twitter](https://twitter.com/rockthejvm). Similar on [LinkedIn](https://linkedin.com/company/rockthejvm). I'm not active on Facebook, although I do have a [page](https://facebook.com/rockthejvm) and I post every once in a while — can't find a rational reason why, I'll fix that.
+
+## 3. YouTube
+
+By far the most compelling impact has been on [YouTube](https://youtube.com/rockthejvm). 3468 subscribers so far!
+
+Besides turning every single article into a video, I also recorded some specials. The [Scala at Light Speed](https://www.youtube.com/watch?v=-8V6bMjThNo&list=PLmtsMNDRU0BxryRX4wiwrTZ661xcp6VPM&ab_channel=RocktheJVM) mini-course was very well received and was watched around 4000 times fully, with another 1000 enrollments on the [main site](https://rockthejvm.com/p/scala-at-light-speed) (tip: you can download it there!). For a starting channel, I'm pretty happy.
+
+I also ran a small experiment. I posted a 3-hour [Java tutorial for beginners](https://www.youtube.com/watch?v=sjGjoDiD2F8&list=PLmtsMNDRU0Bw2dU_Kg_h2SM79afD-pPNh&ab_channel=RocktheJVM) which was shot in a half-screen 8:9 aspect ratio, by the assumption that switching between windows is frustrating if you're watching a tutorial and coding at the same time, so I figured you'd like to have both the video and your code editor visible. Although people liked the content, they didn't scream any "wow I can code much easier with a side-by-side window". So I kept recording my content widescreen.
+
+Some honorable mentions include the Scala [type-level programming](https://www.youtube.com/watch?v=qwUYqv6lKtQ&list=PLmtsMNDRU0ByOQoz6lnihh6CtMrErNax7) mini-series, many videos on [Scala 3](https://www.youtube.com/watch?v=orTmm6OMaLw&list=PLmtsMNDRU0BwsVUbhsH2HMqDMPNhQ0HPc), some [Akka videos](https://www.youtube.com/watch?v=Agze0Ule5_0), concepts that [everybody talks about](https://www.youtube.com/watch?v=UaSXx-oObf4), and comparing tools like [Kafka vs Spark vs Akka](https://www.youtube.com/watch?v=UaSXx-oObf4).
+
+## 4. Courses!
+
+This year, I released courses at a slower pace, whereas 2019 had 8(!) courses. I launched
+
+ - [Spark Streaming](https://rockthejvm.com/p/spark-streaming)
+ - [Spark Optimization](https://rockthejvm.com/p/spark-optimization)
+ - [Spark Performance Tuning](https://rockthejvm.com/p/spark-performance-tuning) aka Spark Optimization 2
+ - [Cats](https://rockthejvm.com/p/cats)
+
+and I've really pushed myself here. The content is difficult and hard to put into concise words, but my students are happy and report many "aha" moments, so I've marked it as a win.
+
+At the same time, I've brought all my Udemy courses to the main website and added [some](https://rockthejvm.com/p/the-scala-bundle) [pretty](https://rockthejvm.com/p/the-akka-bundle) [unbeatable](https://rockthejvm.com/p/the-spark-bundle) [deals](https://rockthejvm.com/p/membership).
+
+The main site right now has ~130 hours of pure video content and >20000 lines of code written on camera, which is pretty bonkers, and there's more incoming!
+
+## 5. Corporate Training
+
+This year, I was very fortunate to land some of the biggest training clients of my (young) career as an instructor. Among them, a Scala/functional programming course at Adobe:
+
+
+
+
+
+
+
+) method.
+- With our consuming implementation in place we have a working publish-subscribe application that generates sensor events to a pulsar topic and a consumer that subscribes to it and consumes those messages.
+- A complete implementation of the consumer can be found [here](https://github.com/polyzos/pulsar-scala-streams/blob/main/src/main/scala/io/ipolyzos/consumers/SensorEventConsumer.scala).
+
+## 5. Apache Pulsar Subscription Types
+
+As mentioned during the blog post, Apache Pulsar provides unification of both messaging and streaming patterns and it does so, by providing different subscription types.
+
+We have the following subscription types:
+
+* **Exclusive Subscription:** only one consumer is allowed to read messages using the subscription at any point in time
+* **Failover Subscription:** only one consumer is allowed to read messages using the subscription at any point in time, but you can have multiple standby consumers to pick up the work in case the active one fails
+* **Shared Subscription:** multiple consumers can be attached to the subscription and the work is shared among them in a Round Robin fashion.
+* **KeyShared Subscription:** multiple consumers can be attached to the subscription and each consumer is assigned a unique set of keys. That consumer is responsible for processing that set of keys it is assigned. In case of failure another consumer will be assigned that set of keys.
+
+
+
+Different Subscription types are used for different use cases. For example in order to implement a typical fan-out messaging pattern you can use the exclusive or failover subscription types. For message queueing and work queues a good candidate is Shared Subscription in order to share the work among multiple consumers. On the other hand for streaming use cases and/or key-based stream processing the Failover and KeyShared subscriptions can be a good candidate that can allow for ordered consumption or scale your processing based on some key.
+
+## 6. Conclusion and Further Reading
+
+In this blog post we made a brief introduction at what Apache Pulsar is, what makes it stand out as a new messaging and streaming platform alternative, how you can create some really simple producing and consuming applications and finally we highlighted how it unified messaging and streaming through different subscription modes.
+
+If this overview sparked your interest and want to learn more, I encourage you to take a look at [Pulsar IO](https://pulsar.apache.org/docs/en/io-overview/) (moves data in and out of Pulsar easily), [Pulsar Functions](http://pulsar.apache.org/docs/en/functions-overview/) (Pulsar’s serverless and lightweight compute framework), which you can use to apply your processing logic on your Pulsar topics, reduce all the boilerplate code needed for producing and consuming applications and combined with [Function Mesh](https://functionmesh.io/) make your event streaming truly cloud-native, by leveraging Kubernetes native features like deployments and autoscaling.
diff --git a/_posts/2021-10-04-type-level-programming-scala-3.md b/_posts/2021-10-04-type-level-programming-scala-3.md
new file mode 100644
index 000000000000..53e7de07ee5b
--- /dev/null
+++ b/_posts/2021-10-04-type-level-programming-scala-3.md
@@ -0,0 +1,191 @@
+---
+title: "Type-Level Programming in Scala 3, Part 1: Comparing Types"
+date: 2021-10-04
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "Learning how to use the power of givens to infer type relationships in Scala 3... at compile time."
+---
+
+This article will introduce you to type-level programming on Scala 3 — the capability to express computations on _types_ instead of values, by making the compiler figure out some type relationships at compile time. By encoding some computational problem as a type restriction or as an enforcement of a type relationship, we can make the compiler "solve" our computation by searching, inferring or validating the proper use of types.
+
+This will be a 2-part series, specifically focused on **Scala 3**. I also published an older [type-level programming miniseries](/type-level-programming-part-1) for Scala 2, which still works on Scala 3 as because implicits are still supported. In this version, we're going to use
+
+- [givens](/scala-3-given-using)
+- generics
+- type members (for the next article)
+
+The final objective of this mini-series is to be able to _sort type-level lists at compile time_. For the sake of variety, in this series we're going to do a _type-level quicksort_ instead of a mergesort (which we did in the Scala 2 miniseries). In this article, we'll take on the much smaller problem of properly comparing numbers (as types), at compile time.
+
+## 1. Encoding Type-Level Numbers
+
+The natural numbers set consists of all the numbers from 0 to infinity, counted one by one. One way of thinking about the naturals set is to imagine all the numbers extending forever. An alternative way — which is also much more useful for us — is to define these numbers in terms of their _relationship_ to each other. This is called the [Peano](https://en.wikipedia.org/wiki/Giuseppe_Peano) representation of numbers.
+
+```scala3
+ trait Nat
+ class _0 extends Nat
+ class Succ[A <: Nat] extends Nat
+```
+
+The lines above fully represent the entire set of natural numbers: the number zero, and the succession relationship between any two numbers. By these two rules alone, we can obtain any number in the naturals set. For example, the number 127 is `Succ[Succ[Succ[...[_0]]]]`, where `Succ` occurs 127 times. For the following examples, we should add a few type aliases:
+
+```scala3
+ type _1 = Succ[_0]
+ type _2 = Succ[_1] // Succ[Succ[_0]]
+ type _3 = Succ[_2] // Succ[Succ[Succ[_0]]]
+ type _4 = Succ[_3] // ... and so on
+ type _5 = Succ[_4]
+```
+
+Note that this notation has nothing to do with the literal types [recently added in Scala 3](/new-types-scala-3). The literal types have no relationship to each other, so we'll need to model this relationship ourselves.
+
+## 2. Ordering Numbers at the Type Level
+
+As in the Scala 2 version of the article, we'll create a small trait to describe the ordering relationship between types, we'll name this `<`, because we can.
+
+```scala3
+ trait <[A <: Nat, B <: Nat]
+```
+
+For example, we'll say that the number 1 is less than 2 _if there exists an instance of_ `<[_1, _2]`. This is how we'll interpret the "less than" relationship, between types, at the type level.
+
+In order to model this relationship, we'll make the compiler prove the truth value of the question "is 1 less than 3" by creating `given` instances of `<`. If the compiler can have access to one — either by our explicit definition or by a synthetic construction by the compiler — then the relationship is proven. If the compiler cannot access or create a `given` instance of the appropriate type, the relationship cannot be proven and therefore is deemed to be false.
+
+## 3. Proving Ordering at the Type Level
+
+First, we need to think about how we can prove this relationship in mathematical terms. How can we determine that `_3` is "less than" `_5` when all we have is the "number" `_0` and the succession relationship between two consecutive numbers?
+
+We can solve this problem by writing the axioms (basic truths) of the `<` relationship. Here is how to think about it:
+
+1. The number 0 is smaller than any other natural number. Besides zero, every other natural number is the successor of some other number. So it's safe to say that, for every number in the naturals set, the successor to that number is greater than zero.
+2. If we can somehow prove that number A is smaller than number B, then it's also true that the successor of A is smaller than the successor of B. This axiom can also be expressed backwards: we can say that A is less than B _if and only if_ the predecessor of A is less than the predecessor of B.
+
+Let's walk through an example: say we want to determine if `_3 < _5`.
+
+- By the second axiom, `_3 < _5` if and only `_2 < _4` because `_3 = Succ[_2]` and `_5 = Succ[_4]`. So we need to prove `_2 < _4`.
+- Again, by the second axiom, `_2 < _4` if and only if `_1 < _3`, for the same reason.
+- Second axiom again: `_1 < _3` if and only if `_0 < _2`.
+- `_0 < _2` is true, by virtue of the first axiom.
+- Therefore, walking back, `_3 < _5` is true.
+
+How can we embed that in Scala code? We'll make the compiler search for, or create, `given` instances for the `<` type.
+
+- For the first axiom, we'll make the compiler generate a given `<[_0, Succ[N]]`, for every `N` which is a natural.
+- For the second axiom, we'll make the compiler generate a given `<[Succ[A], Succ[B]]` if it can find a given `<[A, B]` already in scope.
+
+Getting that in code leads to the following `given`s:
+
+```scala3
+ given basic[B <: Nat]: <[_0, Succ[B]] with {}
+ given inductive[A <: Nat, B <: Nat](using lt: <[A, B]): <[Succ[A], Succ[B]] with {}
+```
+
+Note that in Scala 3.1 (not out yet at the time of writing) you'll be able to say `<[_0, Succ[B]]()` instead of `<[_0, Succ[B]] with {}`, for conciseness.
+
+For ergonomics, we're going to store the above givens in the companion of `<`, so that the code will look like this:
+
+```scala3
+ object < {
+ given basic[B <: Nat]: <[_0, Succ[B]] with {}
+ given inductive[A <: Nat, B <: Nat](using lt: <[A, B]): <[Succ[A], Succ[B]] with {}
+ def apply[A <: Nat, B <: Nat](using lt: <[A, B]) = lt
+ }
+```
+
+I also took the liberty of writing an `apply` method which simply surfaces out whatever `given` value of the requested type the compiler is able to find (or synthesize).
+
+## 4. Testing Type-Level Comparisons
+
+With the above code in place, let's see how the compiler validates (or not) the type relationships. The simple test is: does the code compile?
+
+```scala3
+ val validComparison = <[_3, _5]
+```
+
+This one is valid. The code compiles. Let's see what the compiler does behind the scenes:
+
+- We're requesting a `given` of type `<[_3, _5]`. The compiler needs to find if it can generate one from either of the two `given` synthesizers we specified earlier.
+- The second `given` works, because the compiler looks at the signature of the as-of-yet-ungenerated given: `<[Succ[_2], Succ[_4]]`. However, in order to generate that, it needs to find or create a given of type `<[_2, _4]`.
+- Again, teh second `given` can be used, for the same reason. For that, the compiler needs a given of type `<[_1, _3]`.
+- Same deal, the compiler can create a `given` of type `<[_1, _3]` if it can find a given of type `<[_0, _2]`.
+- Using the _first_ given this time around, we see that the compiler can create a given of type `<[_0, _2]` because `_2 = Succ[_1]`.
+- Walking backwards, the compiler can generate all the intermediate givens in order to create the given we requested.
+
+In other words, the relationship `<[_3, _5]`, or written infix `_3 < _5` can be proven, so it's true. Notice how similarly the compiler generated the given instances in the exact same sequence of steps we used for our mathematical induction above.
+
+By the same mechanism, we can prove any less-than relationships between types that are mathematically correct. The `given`s specification is identical to the mathematical description of the axioms we outlined in the previous section. Conversely, invalid relationships, e.g. `_4 < _2` cannot be proven because the compiler cannot find (or create) a given of the appropriate type, and so it will not compile our code.
+
+## 5. Extending Type-Level Comparisons
+
+I took the liberty of using the same principle to create a "less than or equal" relationship between types. This is going to be useful for the quicksort we're going to do in the upcoming article.
+
+The axioms are similar:
+
+- The number 0 is less than or equal to any other number. In Scala terms, it means we can always generate a given `<=[_0, N]` for any `N <: Nat`.
+- The second axiom of `<=` is identical to the one for `<`: if a number-type `A` is "less than or equal to" another number type `B`, that means that `Succ[A] <= Succ[B]` as well.
+
+Compressed, the code will look like this:
+
+```scala3
+ trait <=[A <: Nat, B <: Nat]
+ object <= {
+ given lteBasic[B <: Nat]: <=[_0, B] with {}
+ given inductive[A <: Nat, B <: Nat](using lte: <=[A, B]): <=[Succ[A], Succ[B]] with {}
+ def apply[A <: Nat, B <: Nat](using lte: <=[A, B]) = lte
+ }
+```
+
+Again, the correct relationships compile:
+
+```scala3
+val lteTest = <=[_3, _3]
+val lteTest2 = <=[_3, _5]
+```
+
+whereas the incorrect relationships do not:
+
+```scala3
+val invalidLte = <=[_5, _3] // cannot find the appropriate given
+```
+
+## 6. Conclusion
+
+The complete code can be found below:
+
+```scala3
+ trait Nat
+ class _0 extends Nat
+ class Succ[N <: Nat] extends Nat
+
+ type _1 = Succ[_0]
+ type _2 = Succ[_1] // Succ[Succ[_0]]
+ type _3 = Succ[_2]
+ type _4 = Succ[_3]
+ type _5 = Succ[_4]
+
+ // name it "LessThan", then refactor to "<"
+ trait <[A <: Nat, B <: Nat]
+ object < {
+ given basic[B <: Nat]: <[_0, Succ[B]] with {}
+ given inductive[A <: Nat, B <: Nat](using lt: <[A, B]): <[Succ[A], Succ[B]] with {}
+ def apply[A <: Nat, B <: Nat](using lt: <[A, B]) = lt
+ }
+ val comparison = <[_1, _3]
+ /*
+ <.apply[_1, _3] -> requires <[_1, _3]
+ inductive[_0, _2] -> requires <[_0, _2]
+ ltBasic[_1] -> produces <[_0, Succ[_1]] == <[_0, _2]
+ */
+
+ trait <=[A <: Nat, B <: Nat]
+ object <= {
+ given lteBasic[B <: Nat]: <=[_0, B] with {}
+ given inductive[A <: Nat, B <: Nat](using lte: <=[A, B]): <=[Succ[A], Succ[B]] with {}
+ def apply[A <: Nat, B <: Nat](using lte: <=[A, B]) = lte
+ }
+ val lteTest: _1 <= _1 = <=[_1, _1]
+```
+
+With this code, we showed how we can compare natural numbers — in their type-level Peano representation — at compile time, by proving the existence of the appropriate `given` value of the comparison "operator" (again, at the type level).
+
+In the next part, we'll take this skill to level 90 and do a quicksort on types.
diff --git a/_posts/2021-10-11-type-level-quicksort.md b/_posts/2021-10-11-type-level-quicksort.md
new file mode 100644
index 000000000000..47ad208a0c12
--- /dev/null
+++ b/_posts/2021-10-11-type-level-quicksort.md
@@ -0,0 +1,408 @@
+---
+title: "Type-Level Programming in Scala 3, Part 2: A Quicksort on Types"
+date: 2021-10-11
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "Level 90 of type-level programming: a real sorting algorithm on lists... as types."
+---
+
+This article is the part 2 of the mini-series of type-level programming in Scala 3. If you want an introduction to type-level programming and a gentle tutorial of how we can encapsulate a computational problem as a relationships between types which the compiler can solve, [read the first part here](/type-level-programming-scala-3).
+
+In this article, we're going to use the skills we learned in the previous part, and take them to level 90: we're going to write a quicksort algorithm on lists, at the type level, which the compiler can solve at compile time.
+
+## 1. Where We Left Off
+
+In the first part of the mini-series focused on the representation of natural numbers as types, in the [Peano arithmetic](https://en.wikipedia.org/wiki/Peano_axioms). The entire set of naturals can be expressed just with the number 0 and the succession relationship between two consecutive numbers.
+
+With this representation, we can demonstrate a "less than" and "less than, or equal" relationship between numerical types by proving the existence of a `given` value which denotes the appropriate relationship: `<` or `<=`. The compiler synthesizes these instances based on rules defined by the `given`s below.
+
+```scala3
+ trait Nat
+ class _0 extends Nat
+ class Succ[N <: Nat] extends Nat
+
+ type _1 = Succ[_0]
+ type _2 = Succ[_1] // Succ[Succ[_0]]
+ type _3 = Succ[_2]
+ type _4 = Succ[_3]
+ type _5 = Succ[_4]
+
+ // name it "LessThan", then refactor to "<"
+ trait <[A <: Nat, B <: Nat]
+ object < {
+ given basic[B <: Nat]: <[_0, Succ[B]] with {}
+ given inductive[A <: Nat, B <: Nat](using lt: <[A, B]): <[Succ[A], Succ[B]] with {}
+ def apply[A <: Nat, B <: Nat](using lt: <[A, B]) = lt
+ }
+ val comparison = <[_1, _3]
+ /*
+ <.apply[_1, _3] -> requires <[_1, _3]
+ inductive[_0, _2] -> requires <[_0, _2]
+ ltBasic[_1] -> produces <[_0, Succ[_1]] == <[_0, _2]
+ */
+
+ trait <=[A <: Nat, B <: Nat]
+ object <= {
+ given lteBasic[B <: Nat]: <=[_0, B] with {}
+ given inductive[A <: Nat, B <: Nat](using lte: <=[A, B]): <=[Succ[A], Succ[B]] with {}
+ def apply[A <: Nat, B <: Nat](using lte: <=[A, B]) = lte
+ }
+ val lteTest: _1 <= _1 = <=[_1, _1]
+```
+
+## 2. A Type-Level List
+
+In the style of [shapeless](https://github.com/milessabin/shapeless), we'll denote a heterogeneous list (HList) at the type level as follows:
+
+```scala3
+ trait HList
+ class HNil extends HList
+ class ::[H <: Nat, T <: HList] extends HList
+```
+
+For example, the list `[1,2,3,4]` at the type level would be analogous to the type `::[_1, ::[_2, ::[_3, ::[_4, HNil]]]]`, or `_1 :: _2 :: _3 :: _4 :: HNil` with the infix notation.
+
+We will attempt to "sort" such an HList using a quicksort on types. Much like testing the comparison between numbers, e.g. `<[_1, _3]`, we will test the "sorted" relationship between two lists. Something like `Sorted[_2 :: _3 :: _1 :: _4 :: HNil, _1 :: _2 :: _3 :: _4 :: HNil]` will test that the sorted version of `_2 :: _3 :: _1 :: _4 :: HNil` is `_1 :: _2 :: _3 :: _4 :: HNil`.
+
+Then we will take this to level 91 and make the compiler _infer_ the appropriate result type for us, automatically.
+
+It's a long way to go. For a "normal" quicksort on a linked list, we'll need to
+
+- take the head of the list (pivot), then _partition_ the list into two parts: one with elements smaller than the pivot, and those greater than or equal to it
+- sort the partitions recursively
+- combine the sorted partitions with concatenation
+
+For the type-level sort, this plan corresponds to 3 type-level operations:
+
+- concatenation
+- partitioning
+- sorting
+
+We'll take them all in turn.
+
+## 3. Type-level Quicksort, Part 1: The Concatenation
+
+In the first part of the mini-series we use type-level programming as "tests". In other words, the relationship `1 < 3` is true only if there is an instance of the type `<[_1, _3]` that the compiler can either find or generate.
+
+We'll do the same with concatenating two lists. We'll use concatenation as a test, which we'll denote `Concat[HA, HB, O]`, where all type members are subtypes of `HList`. For example, the concatenation `[1,2] ++ [3,4] == [1,2,3,4]` is only true if there is an instance of a `Concat[_1 :: _2 :: HNil, _3 :: _4 :: HNil, _1 :: _2 :: _3 :: _4 :: HNil]`. Otherwise the relationship is not true.
+
+The axioms of concatenation are as follows:
+
+1. (basic) Concatenating the empty list with any list `L` results in that list `L`.
+2. (inductive/recursive) Concatenating any non-empty list, say `H :: T`, with any list `L` results in a new list whose head is `H` and tail is the concatenation of `T` and `L`.
+
+Written as `given`s, the first axiom allows the compiler to generate any `Concat[HNil, L, L]` for any list type `L`
+
+```scala3
+ given basicEmpty[L <: HList]: Concat[HNil, L, L] with {}
+```
+
+and the second axiom allows the compiler to _rely on_ an existing (recursive) concatenation before it can synthesize a concatenation for our original lists:
+
+```scala3
+ given inductive[N <: Nat, HA <: HList, HB <: HList, O <: HList](using Concat[HA, HB, O]): Concat[N :: HA, HB, N :: O] with {}
+```
+
+Putting all pieces together, along with a `summon`-like `apply` method in the companion of `Concat`, we'll get this:
+
+```scala3
+ trait Concat[HA <: HList, HB <: HList, O <: HList]
+ object Concat {
+ given basicEmpty[L <: HList]: Concat[HNil, L, L] with {}
+ given inductive[N <: Nat, HA <: HList, HB <: HList, O <: HList](using Concat[HA, HB, O]): Concat[N :: HA, HB, N :: O] with {}
+ def apply[HA <: HList, HB <: HList, O <: HList](using concat: Concat[HA, HB, O]): Concat[HA, HB, O] = concat
+ }
+```
+
+To test this real quick, the concatenation of the lists `[1,2]` and `[3,4]` looks like this:
+
+```scala
+ val concat = Concat[_0 :: _1 :: HNil, _2 :: _3 :: HNil, _0 :: _1 :: _2 :: _3 :: HNil]
+```
+
+The compiler successfully compiles this code because:
+
+- the apply method needs a `Concat[_0 :: _1 :: HNil, _2 :: _3 :: HNil, _0 :: _1 :: _2 :: _3 :: HNil]`
+- the compiler can use the `inductive` method, but it needs a `Concat[_1 :: HNil, _2 :: _3 :: HNil, _1 :: _2 :: _3 :: HNil]`
+- the compiler can again use `inductive` but it needs a `Concat[HNil, _2 :: _3 :: HNil, _2 :: _3 :: HNil]`
+- the compiler can use `basic` with the type `_2 :: _3 :: HNil` to create a `Concat[HNil, _2 :: _3 :: HNil, _2 :: _3 :: HNil]`
+- working backwards, the compiler can create all the necessary intermediate `given`s for the apply method
+
+Conversely, any incorrect concatenation results in uncompilable code.
+
+Part 1 of quicksort, check.
+
+## 4. Type-level Quicksort, Part 2: The Partitioning
+
+The second phase of our type-level quicksort algorithm is a type-level partitioning. Namely, we need to pick a "head" (the pivot) off our type-level linked list and separate the elements "smaller than" and "greater than or equal to" the pivot.
+
+For this step, we will use another type, `Partition[HL <: HList, L <: HList, R <: HList]` which denotes that the list `HL` was successfully partitioned into `L` (the elements smaller than the pivot, including the pivot) and `R` (containing the elements greater than or equal to the pivot).
+
+For example, we'll say that the list `[_2 :: _3 :: _1 :: HNil]` was successfully partitioned into `[_2 :: _1 :: HNil]` and `[_3 :: HNil]` if the compiler can find or synthesize a `given` instance of `Partition[_2 :: _3 :: _1 :: HNil, _2 :: _1 :: HNil, _3 :: HNil]`.
+
+The axioms of partitioning are as follows:
+
+1. Partitioning an empty list results in the empty list on the "left" and the empty list on the "right".
+2. Partitioning a one-element list results in that same list on the "left" and the empty list on the "right".
+3. Considering a list with at least two elements `P :: N :: T` (pivot, next number and tail of the list), we'll need to assume a recursive partitioning of `T` into a "left" `L` (with elements smaller than the pivot) and a "right" `R` (with elements >= the pivot). Now we need to decide where `N` is going to stay.
+
+ - If `P <= N`, then `N` will stay on the "right" side.
+ - If `N < P`, then `N` will stay on the "left" side.
+
+Axioms 1 and 2 are more easily translatable to givens. The first axiom creates a given for a Partition where the original list is HNil and the "left" and "right" side of the split are both HNil.
+
+```scala3
+ given basic: Partition[HNil, HNil, HNil] with {}
+```
+
+The second axiom creates a given for any one-element list: the "left" side will be the list itself, while the "right" side is empty.
+
+```scala3
+ given basic2[N <: Nat]: Partition[N :: HNil, N :: HNil, HNil] with {}
+```
+
+The third axiom has two parts, so we will implement each "branch" separately. We will assume a list with at least two elements, `P :: N :: T`, where P is the pivot, N is a "number" and T is the tail of the list. If we can somehow split T into `L` and `R`, then if `P <= N`, then `N` will stay in the right. The encoding is a bit more complex:
+
+```scala3
+ given inductive[P <: Nat, N <: Nat, T <: HList, L <: HList, R <: HList]
+ (using P <= N, Partition[P :: T, P :: L, R]):
+ Partition[P :: N :: T, P :: L, N :: R] with {}
+```
+
+The other branch is similar, but we'll assume `N < P` this time, which means `N` will stay on the "left":
+
+```scala3
+ given inductive2[P <: Nat, N <: Nat, T <: HList, L <: HList, R <: HList]
+ (using N < P, Partition[P :: T, P :: L, R]):
+ Partition[P :: N :: T, P :: N :: L, R] with {}
+```
+
+That, plus an `apply` method to summon the relevant given, will make `Partition` look like this:
+
+```scala3
+ trait Partition[HL <: HList, L <: HList, R <: HList] // pivot included in left
+ object Partition {
+ given basic: Partition[HNil, HNil, HNil] with {}
+ given basic2[N <: Nat]: Partition[N :: HNil, N :: HNil, HNil] with {}
+
+ given inductive[P <: Nat, N <: Nat, T <: HList, L <: HList, R <: HList]
+ (using P <= N, Partition[P :: T, P :: L, R]):
+ Partition[P :: N :: T, P :: L, N :: R] with {}
+
+ given inductive2[P <: Nat, N <: Nat, T <: HList, L <: HList, R <: HList]
+ (using N < P, Partition[P :: T, P :: L, R]):
+ Partition[P :: N :: T, P :: N :: L, R] with {}
+
+ def apply[HL <: HList, L <: HList, R <: HList]
+ (using partition: Partition[HL, L, R]): Partition[HL, L, R] =
+ partition
+ }
+```
+
+Testing this to see whether our `Partition` type works well:
+
+```scala3
+ val partition = Partition[_2 :: _3 :: _1 :: HNil, _2 :: _1 :: HNil, _3 :: HNil]
+```
+
+This code compiles because the compiler:
+
+- requires a `Partition[_2 :: _3 :: _1 :: HNil, _2 :: _1 :: HNil, _3 :: HNil]`
+- can use `inductive`, but it requires a `Partition[_2 :: _1 :: HNil, _2 :: _1 :: HNil, HNil]`
+- can use `inductive2`, but it requires a `Partition[_2 :: HNil, _2 :: HNil, HNil]`
+- can use `basic2[_2]` to build a `Partition[_2 :: HNil, _2 :: HNil, HNil]`
+- works backwards to build the necessary givens.
+
+Note: there are alternative partitioning schemes which the compiler will not be able to validate. This partitioning does not work, for example, although it's perfectly valid.
+
+```scala3
+ val partitionAlsoValid = Partition[_2 :: _3 :: _1 :: _4 :: HNil, _2 :: _1 :: HNil, _4 :: _3 :: HNil]
+```
+
+But the following does (the `_3` comes before the `_4`)
+
+```scala3
+ val partitionAlsoValid = Partition[_2 :: _3 :: _1 :: _4 :: HNil, _2 :: _1 :: HNil, _3 :: _4 :: HNil] // works
+```
+
+because the relative order of the elements in the list is kept. For the normal quicksort, the relative order of elements inside each partition is not required.
+
+That is of no consequence for correct sorting, as the compiler will be able to find _at least one_ partitioning that works for quicksort.
+
+## 5. Type-Level Quicksort: The Sorting Operation
+
+This will be the final step in our quicksort algorithm at the type level, which will rely on both concatenation and partitioning (the ops we did before).
+
+I think, by now, we got the hang of it. We'll use a new type `QSort[HL, R]` to test whether the sorting of `HL` is in fact `R`. If the compiler can find or generate a `given` of that type that we want to test, then the sorting is correct, otherwise it's not.
+
+The sorting axioms are as follows:
+
+- Sorting an empty list gives us back an empty list.
+- Sorting a non-empty list, say `N :: T` has several requirements:
+ - the correct partitioning of the list into a "left" `N :: L` (remember, the pivot is here) and a "right" `R`
+ - recursive sorting of `L` and `R` into, say `SL` (sorted "left") and `SR` (sorted "right")
+ - the concatenation of the results: `SL` comes first, then `N` the pivot, then `SR`
+
+The first axiom is similar to the basic axioms of the other operations:
+
+```scala3
+ given basicEmpty: QSort[HNil, HNil] with {}
+```
+
+The second axiom is the juice of this entire article.
+
+```scala3
+ given inductive[N <: Nat, T <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList]
+ (
+ using
+ Partition[N :: T, N :: L, R],
+ QSort[L, SL],
+ QSort[R, SR],
+ Concat[SL, N :: SR, O]
+ ): QSort[N :: T, O] with {}
+```
+
+As before, the complete sorting looks like this:
+
+```scala3
+ trait QSort[HL <: HList, Res <: HList]
+ object QSort {
+ given basicEmpty: QSort[HNil, HNil] with {}
+ // given basicOne[N <: Nat]: QSort[N :: HNil, N :: HNil] with {}
+ given inductive[N <: Nat, T <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList]
+ (
+ using
+ Partition[N :: T, N :: L, R],
+ QSort[L, SL],
+ QSort[R, SR],
+ Concat[SL, N :: SR, O]
+ ): QSort[N :: T, O] with {}
+
+ def apply[HL <: HList, Res <: HList](using sort: QSort[HL, Res]): QSort[HL, Res] = sort
+ }
+```
+
+And we can test it on a bunch of examples to see that they work:
+
+```scala3
+ val sortTest = QSort[_3 :: _2 :: _1 :: _4 :: HNil, _1 :: _2 :: _3 :: _4 :: HNil]
+ val sortTest2 = QSort[_1 :: _2 :: _3 :: _4 :: HNil, _1 :: _2 :: _3 :: _4 :: HNil]
+ val sortTest3 = QSort[_4 :: _3 :: _2 :: _1 :: HNil, _1 :: _2 :: _3 :: _4 :: HNil]
+ val sortTest4 = QSort[_4 :: _4 :: _4 :: HNil, _4 :: _4 :: _4 :: HNil]
+```
+
+That's it, folks! Quicksort on types at compile time!
+
+## 6. Type-Level Quicksort... Auto-Inferred
+
+We can take this even further, though. It's one thing for us to "test" that a sorting is correct, but a whole other thing to _make the compiler figure out the correct sorting on its own!_
+
+Let's consider a different kind of sorting type:
+
+```scala3
+ trait Sort[HL <: HList] {
+ type Result <: HList
+ }
+```
+
+Notice we eliminated the second generic type argument, and added the abstract type member instead. We will make the compiler "store" that abstract type member all by itself.
+
+In the companion of `Sort`, we'll define a type alias for a sorting operation which looks similar to the `QSort` above.
+
+```scala3
+ type QSort[HL <: HList, O <: HList] = Sort[HL] { type Result = O }
+```
+
+Why did I do that? In the process of finding the right `given`s, the compiler does type inference anyway. So we can make the compiler infer the generic types automatically in the process of `given` resolution, AND in the meantime write the abstract type argument in `Sort`. Here's how we'll rewrite the axioms.
+
+The first axiom is for sorting an empty list:
+
+```scala3
+ given basicEmpty: QSort[HNil, HNil] = new Sort[HNil] { type Result = HNil }
+```
+
+Notice the difference? Instead of creating a `QSort[HNil, HNil] with {}` — which we can't do with abstract type members — we're instead _providing_ the type signature of the `given` (for later resolutions) and instead we use a `Sort[HNil]` with the right type member.
+
+The inductive axiom is more complex, as expected:
+
+```scala3
+ given inductive[N <: Nat, T <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList] (
+ using
+ Partition[N :: T, N :: L, R],
+ QSort[L, SL],
+ QSort[R, SR],
+ Concat[SL, N :: SR, O]
+ ): QSort[N :: T, O] = new Sort[N :: T] { type Result = O }
+```
+
+Again, the only thing that's changed here is how the `QSort` instance is actually created: as an instance of `Sort` with the right types.
+
+In total, the new auto-inferred sort looks like this:
+
+```scala3
+ trait Sort[HL <: HList] {
+ type Result <: HList
+ }
+ object Sort {
+ type QSort[HL <: HList, O <: HList] = Sort[HL] { type Result = O }
+ given basicEmpty: QSort[HNil, HNil] = new Sort[HNil] { type Result = HNil }
+ given inductive[N <: Nat, T <: HList, L <: HList, R <: HList, SL <: HList, SR <: HList, O <: HList] (
+ using
+ Partition[N :: T, N :: L, R],
+ QSort[L, SL],
+ QSort[R, SR],
+ Concat[SL, N :: SR, O]
+ ): QSort[N :: T, O] = new Sort[N :: T] { type Result = O }
+
+ def apply[L <: HList](using sort: Sort[L]): QSort[L, sort.Result] = sort
+ }
+```
+
+Now we can safely test it...
+
+```scala3
+ val qsort = Sort.apply[_4 :: _3 :: _5 :: _1 :: _2 :: HNil]
+```
+
+... or can we?
+
+What's the resulting type? Of course, we can specify it ourselves in the `val` definition, but it defeats the purpose. And the compiler knows the result type anyway!
+
+We could print it to show it's the right one - but we can't simply `.toString` the value, because the generic types, as well as abstract type members, are erased.
+
+We need to save the types to runtime. Rob Norris has this amazing tiny (20-line tiny) [library](https://github.com/tpolecat/typename/tree/main/src/main) that can do that, with a bit of metaprogramming. Adding the following to `build.sbt`
+
+```scala3
+ libraryDependencies += "org.tpolecat" %% "typename" % "1.0.0"
+```
+
+and with a helper method on my part
+
+```scala3
+ def printType[A](value: A)(using typename: TypeName[A]): String = typename.value
+```
+
+we can safely print
+
+```scala3
+ printType(qsort).replaceAll("com.rockthejvm.blog.typelevelprogramming.TypeLevelProgramming.", "") // remove fully qualified type name prefixes
+```
+
+And lo and behold:
+
+```scala3
+ Sort[::[_4, ::[_3, ::[_5, ::[_1, ::[_2, HNil]]]]]] {
+ type Result >:
+ ::[Succ[_0], ::[Succ[_1], ::[Succ[_2], ::[Succ[_3], ::[Succ[_4], HNil]]]]]
+ <: ::[Succ[_0], ::[Succ[_1], ::[Succ[_2], ::[Succ[_3], ::[Succ[_4], HNil]]]]]
+ }
+```
+
+In other words, the result type is `::[Succ[_0], ::[Succ[_1], ::[Succ[_2], ::[Succ[_3], ::[Succ[_4], HNil]]]]]` or `_1 :: _2 :: _4 :: _3 :: _5 :: HNil`!
+
+## 6. Conclusion
+
+If this is not the most badass quicksort you've ever seen in Scala, you can't be human.
+
+Scala rocks, folks.
diff --git a/_posts/2021-10-21-kafka-streams.md b/_posts/2021-10-21-kafka-streams.md
new file mode 100644
index 000000000000..fbf65b46425f
--- /dev/null
+++ b/_posts/2021-10-21-kafka-streams.md
@@ -0,0 +1,928 @@
+---
+title: "Kafka Streams 101"
+date: 2021-10-21
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kafka]
+excerpt: "Apache Kafka is clearly the leading technology concerning message brokers, and Kafka Streams brings a complete stateful streaming system based directly on top of it. Let's see its primary features."
+---
+
+_Another great round by [Riccardo Cardin](https://github.com/rcardin), now a frequent contributor to the Rock the JVM blog. Riccardo is a senior developer, a teacher and a passionate technical blogger. He's well versed with Apache Kafka; he recently published an article on [how to integrate ZIO and Kafka](/zio-kafka). So now he rolled up his sleeves in the quest of writing **the ultimate end-to-end tutorial on Kafka Streams**. We hope you enjoy it! Enter Riccardo:_
+
+Apache Kafka nowadays is clearly the leading technology concerning message brokers. It's scalable, resilient, and easy to use. Moreover, it leverages a bunch of exciting client libraries that offer a vast set of additional features. One of these libraries is _Kafka Streams_.
+
+Kafka Streams brings a complete stateful streaming system based directly on top of Kafka. Moreover, it introduces many exciting concepts, like the duality between topics and database tables. Implementing such ideas, Kafka Streams provides us many valuable operations on topics, such as joins, grouping capabilities, and so on.
+
+Because the Kafka Streams library is quite complex, this article will introduce only its main features, such as the architecture, the Stream DSL with its basic types `KStream`, `KTable`, and `GlobalKTable`, and the transformations defined on them.
+
+## 1. Set Up
+
+As we said, the Kafka Streams library is implemented using a set of client libraries. In addition, we will use the Circe library to deal with JSON messages. Using Scala as the language to do some experiments, we have to declare the following dependencies in the `build.sbt` file:
+
+```scala
+libraryDependencies ++= Seq(
+ "org.apache.kafka" % "kafka-clients" % "2.8.0",
+ "org.apache.kafka" % "kafka-streams" % "2.8.0",
+ "org.apache.kafka" %% "kafka-streams-scala" % "2.8.0",
+ "io.circe" %% "circe-core" % "0.14.1",
+ "io.circe" %% "circe-generic" % "0.14.1",
+ "io.circe" %% "circe-parser" % "0.14.1"
+)
+```
+
+Among the dependencies, we find the `kafka-streams-scala` libraries, a Scala wrapper built around the Java `kafka-streams` library. In fact, using implicit resolution, the tailored Scala library avoids some boilerplate code.
+
+All the examples we'll use share the following imports:
+
+```scala
+import io.circe.generic.auto._
+import io.circe.parser._
+import io.circe.syntax._
+import io.circe.{Decoder, Encoder}
+import org.apache.kafka.common.serialization.Serde
+import org.apache.kafka.streams.kstream.{GlobalKTable, JoinWindows, TimeWindows, Windowed}
+import org.apache.kafka.streams.scala.ImplicitConversions._
+import org.apache.kafka.streams.scala._
+import org.apache.kafka.streams.scala.kstream.{KGroupedStream, KStream, KTable}
+import org.apache.kafka.streams.scala.serialization.Serdes
+import org.apache.kafka.streams.scala.serialization.Serdes._
+import org.apache.kafka.streams.{KafkaStreams, StreamsConfig, Topology}
+
+import java.time.Duration
+import java.time.temporal.ChronoUnit
+import java.util.Properties
+import scala.concurrent.duration._
+```
+
+We will use version 2.8.0 of Kafka. As we've done in the article [ZIO Kafka: A Practical Streaming Tutorial](https://blog.rockthejvm.com/zio-kafka/), we will start the Kafka broker using a Docker container, declared through a `docker-compose.yml` file:
+
+```yaml
+version: '2'
+services:
+ zookeeper:
+ image: confluentinc/cp-zookeeper:6.2.0
+ hostname: zookeeper
+ container_name: zookeeper
+ ports:
+ - "2181:2181"
+ environment:
+ ZOOKEEPER_CLIENT_PORT: 2181
+ ZOOKEEPER_TICK_TIME: 2000
+
+ kafka:
+ image: confluentinc/cp-kafka:6.2.0
+ hostname: broker
+ container_name: broker
+ depends_on:
+ - zookeeper
+ ports:
+ - "29092:29092"
+ - "9092:9092"
+ - "9101:9101"
+ environment:
+ KAFKA_BROKER_ID: 1
+ KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+ KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+ KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+ KAFKA_JMX_PORT: 9101
+ KAFKA_JMX_HOSTNAME: localhost
+```
+
+Please, refer to the above article for further details on starting the Kafka broker inside Docker.
+
+As usual, we need a use case to work with. We'll try to model some functions concerning the management of orders in an e-commerce site. During the process, we will use the following types:
+
+```scala
+object Domain {
+ type UserId = String
+ type Profile = String
+ type Product = String
+ type OrderId = String
+
+ case class Order(orderId: OrderId, user: UserId, products: List[Product], amount: Double)
+ case class Discount(profile: Profile, amount: Double) // in percentage points
+ case class Payment(orderId: OrderId, status: String)
+}
+```
+
+To set up the application, we also need to create the Kafka topics we will use. In Scala, we represent the topics' names as constants:
+
+```scala
+object Topics {
+ final val OrdersByUserTopic = "orders-by-user"
+ final val DiscountProfilesByUserTopic = "discount-profiles-by-user"
+ final val DiscountsTopic = "discounts"
+ final val OrdersTopic = "orders"
+ final val PaymentsTopic = "payments"
+ final val PaidOrdersTopic = "paid-orders"
+}
+```
+
+To create such topics in the broker, we use directly the Kafka clients libraries contained in the Docker image:
+
+```shell
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic orders-by-user \
+ --create
+
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic discount-profiles-by-user \
+ --create \
+ --config "cleanup.policy=compact"
+
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic discounts \
+ --create \
+ --config "cleanup.policy=compact"
+
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic orders \
+ --create
+
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic payments \
+ --create
+
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic paid-orders \
+ --create
+```
+
+As we can see, we defined some topics as `compact`. They are a particular type of topic, and we will introduce them deeper during the article.
+
+## 2. Basics
+
+As we said, the Kafka Streams library is a client library, and it manages streams of messages, reading them from topics and writing the results to different topics.
+
+As we should know, we build streaming applications around three concepts: sources, flows (or pipes), and sinks. Often, we represent streams as a series of tokens, generated by a source, transformed by flows and consumed by sinks:
+
+
+
+A sources generate the elements that are handled in the stream - here they are the messages read from the topic.
+
+A flow is nothing more than a transformation applied to every token. In functional programming, we represent flows using functions such as `map`, `filter`, `flatMap`, and so on.
+
+Last but not least, a sink is where messages are consumed. After a sink, they don't exist anymore. In Kafka Streams, sinks can consume messages to a Kafka topic or use anything other technology (i.e., the standard output, a database, etc.)
+
+In Kafka Stream's jargon, both source, flow, and sink are called _stream processors_. A streaming application is nothing more than a graph where each node is a processor, and edges are called _streams_. We can call such a graph a _topology_.
+
+
+
+So, with these bullets in our Kafka gun, let's proceed diving a little deeper into how we can implement some functionalities of our use case scenario using the Kafka Streams library.
+
+## 3. Messages Serialization and Deserialization
+
+If we want to create any structure on top of Kafka topics, such as stream, we need a standard way to serialize objects into a topic and deserialize messages from topic to objects. The Kafka Streams library uses the so-called `Serde` type.
+
+What's a `Serde`? The `Serde` word stands for `Serializer` and `Deserializer`. A `Serde` provides the logic to read and write a message from and to a Kafka topic.
+
+So, if we have a `Serde[R]` instance, we can deserialize and serialize objects of the type `R`. In this article, we will use JSON format for the payload of Kafka messages. In Scala, one of the most used libraries to marshall and unmarshall JSON into objects is Circe. We already talked about Circe in the post [Unleashing the Power of HTTP Apis: The Http4s Library](https://blog.rockthejvm.com/http4s-tutorial/), when we used it together with the Http4s library.
+
+```scala
+// Scala Kafka Streams library
+object Serdes {
+ implicit def stringSerde: Serde[String]
+
+ implicit def longSerde: Serde[Long]
+
+ implicit def javaLongSerde: Serde[java.lang.Long]
+
+ // ...
+}
+```
+
+In addition, the `Serdes` object defines the function `fromFn`, which we can use to build our custom instance of a `Serde`:
+
+```scala
+// Scala Kafka Streams library
+def fromFn[T >: Null](serializer: T => Array[Byte], deserializer: Array[Byte] => Option[T]): Serde[T]
+```
+
+Wiring all the information together, we can use the above function to create a `Serde` using Circe:
+
+```scala
+object Implicits {
+ implicit def serde[A >: Null : Decoder : Encoder]: Serde[A] = {
+ val serializer = (a: A) => a.asJson.noSpaces.getBytes
+ val deserializer = (aAsBytes: Array[Byte]) => {
+ val aAsString = new String(aAsBytes)
+ val aOrError = decode[A](aAsString)
+ aOrError match {
+ case Right(a) => Option(a)
+ case Left(error) =>
+ println(s"There was an error converting the message $aOrError, $error")
+ Option.empty
+ }
+ }
+ Serdes.fromFn[A](serializer, deserializer)
+ }
+}
+```
+
+The `serde` function constraints the type `A` to have a Circe `Decoder` and an `Encoder` implicitly defined in the scope. Then, it uses the type class `Encoder[A]` to create a JSON string:
+
+```scala
+a.asJson
+```
+
+Moreover, the function uses the type class `Decoder[A]` to parse a JSON string into an object:
+
+```scala
+decode[A](aAsString)
+```
+
+Fortunately, we can autogenerate Circe `Encoder` and `Decoder` type classes importing `io.circe.generic.auto._`.
+
+Now that we presented the library's types to write and read from Kafka topics and created some utility functions to deal with such types, we can build our first stream topology.
+
+## 4. Creating the Topology
+
+First, we need to define the topology of our streaming application. We will use the _Stream DSL_. This DSL, built on top of the low-level [Processor API](https://docs.confluent.io/platform/current/streams/developer-guide/processor-api.html#streams-developer-guide-processor-api), is easier to use and master, having a declarative approach. Using the Stream DSL, we don't have to deal with stream processor nodes directly. The Kafka Streams library will create for us the best processors' topology reflecting the operation with need.
+
+To create a topology, we need an instance of the builder type provided by the library:
+
+```scala
+val builder = new StreamsBuilder
+```
+
+The builder lets us create the Stream DSL's primary types, which are the`KStream`, `Ktable`, and `GlobalKTable` types. Let's see how.
+
+### 4.1. Building a `KStream`
+
+To start, we need to define a source, which will read incoming messages from the Kafka topic `orders-by-user` we created. Unlike other streaming libraries, such as Akka Streams, the Kafka Streams library doesn't define any specific type for sources, pipes, and sinks:
+
+```scala
+val usersOrdersStreams: KStream[UserId, Order] = builder.stream[UserId, Order](OrdersByUserTopic)
+```
+
+We introduced the first notable citizen of the Kafka Streams library: the `KStream[K, V]` type, representing a regular stream of Kafka messages. Each message has a key of type `K` and a value of type `V`.
+
+Moreover, the API to build a new stream looks pretty straightforward because there is a lot of "implicit magic" under the hood. In fact, the full signature of the `stream` methods is the following:
+
+```scala
+// Scala Kafka Streams library
+def stream[K, V](topic: String)(implicit consumed: Consumed[K, V]): KStream[K, V]
+```
+
+You may wonder what the heck a `Consumed[K, V]` is. Well, it's the Java way to provide a `Serde` for the keys and values of Kafka messages to the stream. Having defined the `serde` function, we can straightforwardly build a `Serde` for our `Order` class. We usually put such classes in the companion object:
+
+```scala
+object Order {
+ implicit val orderSerde: Serde[Order] = serde[Order]
+}
+```
+
+However, we can go even further with the implicit generation of the `Serde` class. In fact, if we define the previous `serde` function as `implicit`, the Scala compiler will automatically generate the `orderSerde` since the `Order` class fulfills all the needed context bounds.
+
+So, just as a recap, the following implicit resolution took place:
+
+```
+Order => Decoder[Order] / Encoder[Order] => Serde[Order] => Consume[Order]
+```
+
+Why do we need `Serde` types to be implicit? The main reason is that the Scala Kafka Streams provides the object `ImplicitConversions`. Inside this `object`, we find a lot of useful conversion functions that, given `Serde` objects, let us define a lot of other types, such as the above `Consumed`. Again, all these conversions save us writing a lot of boilerplate code, which we should have written in Java, for example.
+
+As we said, a `KStream[K, V]` represents a stream of Kafka messages. This type defines many valuable functions, which we can group into two different families: stateless transformations and stateful transformations. While the former uses only in-memory data structures, the latter requires saving some information inside the so-called _state store_. We will look at transformations in a minute. But first, we need to introduce the other two basic types of the Stream DSL.
+
+### 4.2. Building `KTable` and `GlobalKTable`
+
+The Kafka Streams library also offers `KTable` and `GlobalKTable`, built both on top of a _compacted topic_. We can think of a compacted topic as a table, indexed by the messages' key. The broker doesn't delete messages in a compacted topic using a time to live policy. Every time a new message arrives, a "row" is added to the "table" if the key was not present, or the value associated with the key is updated otherwise. To delete a "row" from the "table", we just send to the topic a `null` value associated with the selected key.
+
+As we said in section 1, to make a topic compacted, we need to specify it during its creation:
+
+```shell
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic discount-profiles-by-user \
+ --create \
+ --config "cleanup.policy=compact"
+```
+
+The above topic will be the starting point to extend our Kafka Streams application. In fact, its messages have a `UserId` as key and a discount profile as value. A discount profile tells each user which discounts the e-commerce site could apply to the orders of a user. For the sake of simplicity, we represent profiles as simple `String`:
+
+```scala
+type Profile = String
+```
+
+Creating a `KTable` is easy. For example, let's make a `KTable` on top of the `discount-profiles-by-user` topic. Returning to our example, as the users' number of our e-commerce might be high, we need to partition the information among the nodes of the Kafka cluster. So, let's create the `KTable`:
+
+```scala
+final val DiscountProfilesByUserTopic = "discount-profiles-by-user"
+
+val userProfilesTable: KTable[UserId, Profile] =
+ builder.table[UserId, Profile](DiscountProfilesByUserTopic)
+```
+
+As you can imagine, there is more behind the scene than what we can see. Again, using the chain of implicit conversions, the Scala Kafka Streams library creates an instance of the `Consumed` class, which is mainly used to pass `Serde` around. In this case, we use the `Serdes.stringSerde` implicit object, both for the key and the topic's value.
+
+The methods defined on the `KTable` type are more or less the same as those on a `KStream`. In addition, a `KTable` can be easily converted into a `KStream` using the following method (or one of its variants):
+
+```scala
+// Scala Kafka Streams library
+def toStream: KStream[K, V]
+```
+
+As we can imagine, creating a `GlobalKTable` is easy as well. We only need a compacted topic containing a number of keys that is affordable for each cluster node:
+
+```shell
+kafka-topics \
+ --bootstrap-server localhost:9092 \
+ --topic discounts \
+ --create \
+ --config "cleanup.policy=compact"
+```
+
+The number of different instances of discount `Profile` is low. So. let's create a `GlobalKTable` on top of a topic, mapping each discount profile to a discount. First, we define the type modeling a discount:
+
+```scala
+final val DiscountsTopic = "discounts"
+
+case class Discount(profile: Profile, amount: Double)
+```
+
+Then, we can create an instance of the needed `GlobalKTable`:
+
+```scala
+val discountProfilesGTable: GlobalKTable[Profile, Discount] =
+ builder.globalTable[Profile, Discount](DiscountsTopic)
+```
+
+Again, under the hood, the library creates an instance of a `Consumed` object.
+
+So, which is the difference between a `KTable` and a `GlobalKTable`? The difference is that a `KTable` is partitioned between the nodes of the Kafka cluster. However, every node of the cluster receives a full copy of a `GlobalKTable`. So, be careful with `GlobalKTable`.
+
+The `GlobalKTable` type doesn't define any method. So, why should we ever create an instance of a `GlobalKTable`? The answer is in the word "joins". But first, we need to introduce streams transformations.
+
+## 5. Streams Transformations
+
+Once obtained a `KStream` or a `KTable`, we can transform the information they contain using _transformations_. The Kafka Streams library offers two kinds of transformations: stateless and stateful. While the former executes only in memory, the latter requires managing a state to perform.
+
+### 5.1. Stateless Transformations
+
+In the group of stateless transformations, we find the classic functions defined on streams, such as `filter`, `map`, `flatMap`, etc. Say, for example, that we want to filter all the orders with an amount greater than 1,000.00 Euro. We can use the `filter` function (the library also provides a valuable function `filterNot`):
+
+```scala
+val expensiveOrders: KStream[UserId, Order] = usersOrdersStreams.filter { (userId, order) =>
+ order.amount >= 1000
+}
+```
+
+Instead, let's say that we want to extract a stream of all the purchased products, maintaining the `UserId` as the message key. Since we want to map only the values of the Kafka messages, we can use the `mapValues` function:
+
+```scala
+val purchasedListOfProductsStream: KStream[UserId, List[Product]] = usersOrdersStreams.mapValues { order =>
+ order.products
+}
+```
+
+Going further, we can obtain a `KStream[UserId, Product]` instead, just using the `flatMapValues` function:
+
+```scala
+val purchasedProductsStream: KStream[UserId, Product] = usersOrdersStreams.flatMapValues { order =>
+ order.products
+}
+```
+
+Moreover, the library contains stateless terminal transformations, called sinks, representing a terminal node of our stream topology. We can apply no other function to the stream after a sink is reached. Two examples of sink processors are the `foreach` and `to` methods.
+
+The `foreach` method applies to a stream a given function:
+
+```scala
+// Scala Kafka Streams library
+def foreach(action: (K, V) => Unit): Unit
+```
+
+As an example, imagine we want to print all the products purchased by a user. We can call the ` foreach` method directly on the `purchasedProductsStream` stream:
+
+```scala
+purchasedProductsStream.foreach { (userId, product) =>
+ println(s"The user $userId purchased the product $product")
+}
+```
+
+Another interesting sink processor is the `to` method, which persists the messages of the stream into a new topic:
+
+```scala
+expensiveOrders.to("suspicious-orders")
+```
+
+In the above example, we are writing all the orders greater than 1,000.00 Euro in a dedicated topic, probably performing fraud analysis. Again, the Scala Kafka Streams library saves us from typing a lot of code. In fact, the full signature of the `to` method is the following:
+
+```scala
+// Scala Kafka Streams library
+def to(topic: String)(implicit produced: Produced[K, V]): Unit
+```
+
+The implicit instance of the `Produced` type, which is a wrapper around key and value `Serde`, is produced automatically by the functions in the `ImplicitConversions` object, plus our `serde` implicit function.
+
+Last but not least, we have grouping, which groups different values under the same key. We group values maintaining the original key using the `groupByKey` transformation:
+
+```scala
+// Scala Kafka Streams library
+def groupByKey(implicit grouped: Grouped[K, V]): KGroupedStream[K, V]
+```
+
+As usual, the `Grouped` object carries the `Serde` types for keys and values, and it's automatically derived by the compiler if we use the Scala Kafka Streams library.
+
+For example, imagine we want to group the `purchasedProductsStream` to perform some aggregated operation later. In detail, we want to group each user with the purchased products:
+
+```scala
+val productsPurchasedByUsers: KGroupedStream[UserId, Product] = purchasedProductsStream.groupByKey
+```
+
+As we may notice, we introduced a new type of stream, the `KGroupedStream`. This type defines only stateful transformation on it. So, for this reason, we say that grouping is the precondition to stateful transformations.
+
+Moreover, if we want to group a stream using something that is not the messages' keys, we can use the `groupBy` transformation:
+
+```scala
+val purchasedByFirstLetter: KGroupedStream[String, Product] =
+ purchasedProductsStream.groupBy[String] { (userId, products) =>
+ userId.charAt(0).toLower.toString
+ }
+```
+
+In the above example, we group products by the first letter of the `UserId` of the user who purchased them. It seems a harmless operation from the code, as we are only changing the stream's key. However, since Kafka partitioned topics by key, the change of the key marks the topic for re-partitioning.
+
+So, suppose the marked stream will be materialized in a topic or in a state store (more to come on state stores) by the following transformation. In that case, the contained messages will be potentially moved to another node of the Kafka cluster. Re-partitioning is an operation that should be done with caution because it could generate a heavy network load.
+
+### 5.2. Stateful Transformations
+
+As the name of these types of transformations suggested, the Kafka Streams library needs to maintain some kind of state to manage them, and it's called _state store_. The state store, which is automatically controlled by the library if we use the Stream DSL, can be an in-memory hashmap or an instance of [RocksDB](http://rocksdb.org/), or any other convenient data structure.
+
+Each state store is local to the node containing the instance of the stream application and refers to the messages concerning the partitions owned by the node. So, the global state of a stream application is the sum of all the states of the single node. Kafka Streams offers fault-tolerance and automatic recovery for local state stores.
+
+Now that we know about the existence of state stores, we can start talking about stateful transformations. There are many of them, such as:
+
+- Aggregations
+- Aggregations using windowing
+- Joins
+
+Joins are considered stateful transformations too, but we will treat joins in a dedicated section. However, we can make some examples of aggregations. Aggregations are key-based operations, which means that they always operate over records of the same key.
+
+As we saw, we previously obtained a `KGroupedStream` containing the products purchased by users:
+
+```scala
+val productsPurchasedByUsers: KGroupedStream[UserId, Product] = purchasedProductsStream.groupByKey
+```
+
+Now, we can count how many products each user purchased by calling the `count` transformation:
+
+```scala
+val numberOfProductsByUser: KTable[UserId, Long] = productsPurchasedByUsers.count()
+```
+
+Since the number of purchased products by users updates every time a new message is available, the result of the `count` transformation is a `KTable`, which will update during time accordingly.
+
+The `count` transformation uses the implicit parameters' resolution we just saw. In fact, its signature is the following:
+
+```scala
+// Scala Kafka Stream library
+def count()(implicit materialized: Materialized[K, Long, ByteArrayKeyValueStore]): KTable[K, Long]
+```
+
+As for the `Consumed` implicit objects, the implicit `Materialized[K, V, S]` instance is directly derived by the compiler from the available implicit instances of key and value `Serde`.
+
+The `count` transformation is not the only aggregation type in the Kafka Streams library, offering generic aggregations. Since simple aggregations are very similar to the example we associated with the `count` transformation, we now introduce _windowed aggregations_ instead.
+
+In detail, the Kafka Streams library lets us aggregate messages using a time window. All the messages that arrived inside the window are eligible for being aggregated. Clearly, we are talking about a sliding window through time. The library allows us to aggregate using different types of windows, each one with its own features. Since [windowing](https://docs.confluent.io/platform/current/streams/developer-guide/dsl-api.html#streams-developer-guide-dsl-windowing) is a complex issue, we will not go deeper into it in this article.
+
+For our example, we will use _Tumbling time windows_. They model fixed-size, non-overlapping, gap-less windows. In detail, we want to know how many products our users purchase every ten seconds. First, we need to create the window representation:
+
+```scala
+val everyTenSeconds: TimeWindows = TimeWindows.of(10.second.toJava)
+```
+
+Then, we use it to define our windowed aggregation:
+
+```scala
+val numberOfProductsByUserEveryTenSeconds: KTable[Windowed[UserId], Long] =
+ productsPurchasedByUsers.windowedBy(everyTenSeconds)
+ .aggregate[Long](0L) { (userId, product, counter) =>
+ counter + 1
+ }
+```
+
+As for the `count` transformation, the final result is a `Ktable`. However, this time we have a `Windowed[UserId]` as the key type, a convenient type containing both the key and the lower and upper bound of the window.
+
+The Scala Kafka Streams library defines the `aggregate` transformation as the Scala language defines the `foldLeft` method on sequences. The first parameter is the starting accumulation point, and the second is the folding function. Finally, an implicit instance of a `Materialized[K, V, S]` object is automatically derived by the compiler:
+
+```scala
+// Scala Kafka Stream library
+def aggregate[VR](initializer: => VR)(aggregator: (K, V, VR) => VR)(
+ implicit materialized: Materialized[K, VR, ByteArrayWindowStore]
+): KTable[Windowed[K], VR]
+```
+
+The library defines many other stateful transformations. Please, refer to the [official documentation](https://kafka.apache.org/28/documentation/streams/developer-guide/dsl-api.html#stateful-transformations) that lists all of them.
+
+## 6. Joining Streams
+
+In my opinion, the most essential feature of the Kafka Streams library is the ability to join streams. The Kafka team strongly supports the [duality between streams and database tables](https://docs.confluent.io/platform/current/streams/concepts.html#duality-of-streams-and-tables). To keep it simple, we can view a stream as the changelog of a database table, whose primary keys are equal to the keys of the Kafka messages.
+
+Following this duality, we can think about records in a `KStream` as they are INSERT operations on a table. In fact, for the nature of a `KStream`, every message is different from any previous message. Instead, a `KTable` is an abstraction of a changelog stream, where each record represents an UPSERT: If the key is not present in the table, the record is equal to an INSERT, and an UPDATE otherwise.
+
+With these concepts in mind, it's easier for us to accept the existence of joins between Kafka Streams.
+
+As we already said, joins are stateful operations, requiring a state store to execute.
+
+### 6.1. Joining a `KStream` and a `KTable`
+
+The most accessible kind of join is between a `KStream` and a `KTable`. The join operation is on the keys of the messages. The broker has to ensure that the data is co-partitioned. To go deeper into co-partitioning, please refer to [Joining](https://docs.confluent.io/platform/current/streams/developer-guide/dsl-api.html#joining). To put it simply, if data of two topics are co-partitioned, then the Kafka broker can ensure the joining message resides on the same node of the cluster, avoiding shuffling of messages between nodes.
+
+Returning to our leading example, imagine we want to join the orders stream, which is indexed by `UserId`, with the table containing the discount profile of each user:
+
+```scala
+val ordersWithUserProfileStream: KStream[UserId, (Order, Profile)] =
+ usersOrdersStreams.join[Profile, (Order, Profile)](userProfilesTable) { (order, profile) =>
+ (order, profile)
+ }
+```
+
+As we have seen in many cases, the Scala Kafka Streams library saves us from typing a lot of boilerplate code, implicitly deriving the type that carries the `Serde` information:
+
+```scala
+// Scala Kafka Stream library
+def join[VT, VR](table: KTable[K, VT])(joiner: (V, VT) => VR)(implicit joined: Joined[K, V, VT]): KStream[K, VR]
+```
+
+The first method's parameter is the `KTable`, whereas the second is a function that returns a new value of any type given the pair of the joined values.
+
+In our use case, the join produces a stream containing all the orders purchased by each user, added with the discount profile information. So, the result of a join is a set of messages having the same key as the originals and a transformation of the joined messages' payloads as value.
+
+### 6.2. Joining with a `GlobalKTable`
+
+Another type of join is between a `KStream` (or a `KTable`) and a `GlobalKTable`. As we said, the broker replicates the information of a `GlobalKTable` in each cluster node. So, we don't need the co-partitioning property anymore because the broker ensures the locality of `GlobalKTable` messages for all the nodes.
+
+In fact, the signature of this type of `join` transformation is different from the previous:
+
+```scala
+// Scala Kafka Stream library
+def join[GK, GV, RV](globalKTable: GlobalKTable[GK, GV])(
+ keyValueMapper: (K, V) => GK,
+ joiner: (V, GV) => RV,
+): KStream[K, RV]
+```
+
+The `keyValueMapper` input function maps the information of the stream in the key `GK` of `GlobalKTable`. In this case, we can use any helpful information in the message, both the key and the payload. Instead, the transformation uses the `joiner` function to extract the new payload.
+
+In our example, we can use a join between the stream `ordersWithUserProfileStream` and the global table `discountProfilesGTable` to obtain a new stream with the amount of the discounted order, using the discount associated with the profile of a `UserId`:
+
+```scala
+val discountedOrdersStream: KStream[UserId, Order] =
+ ordersWithUserProfileStream.join[Profile, Discount, Order](discountProfilesGTable)(
+ { case (_, (_, profile)) => profile }, // Joining key
+ { case ((order, _), discount) => order.copy(amount = order.amount * discount.amount) }
+ )
+```
+
+Obtaining the joining key is easy: We just select the `Profile` information contained in the messages' payload of the stream `ordersWithUserProfileStream`. Then, the new value of each message is the discounted amount.
+
+### 6.3. Joining `KStreams`
+
+Last but not least, we have another type of join transformation, maybe the most interesting. We're talking about the join between two streams.
+
+In the joins we've seen so far, one of the two joining operands always represented a table, which means a persistent form of information. Once a key enters the table, it will be present in it until someone removes it. So, Joining with a table reduces to make a lookup in table's keys for every message in the stream.
+
+However, streams are continuously changing pieces of information. So, how can we join such volatile data? Once again, windowing comes into help. The join transformation between two streams joins messages by key. Moreover, messages must arrive at the topics within a sliding window of time. Besides, the data must be co-partitioned for the join between `KStream` and `KTable`.
+
+Since joins are stateful transformations, we must window the join between two streams; otherwise, the underlining state store would grow indefinitely.
+
+Moreover, the semantics of stream-stream join is that a new input record on one side will produce a join output for each matching record on the other side, and there can be multiple such matching records in a given join window.
+
+Talking about purchased orders, imagine we want to join the stream of discounted orders with stream listing orders that received payment, obtaining a stream of paid orders. First, we need to define the latter stream built upon the topic called `payments`. Moreover, we define the `Payment` type that associates each `OrderId` with its payment status:
+
+```scala
+val paymentsStream: KStream[OrderId, Payment] = builder.stream[OrderId, Payment](PaymentsTopic)
+```
+
+It's reasonable that the payment of the order arrives at most some minutes after the order itself. So, it seems that we have a perfect use case to apply the join between streams. But first, we need to change the key of the `discountedOrdersStream` to an `OrderId`. Fortunately, the library offers the transformation called `selectKey`:
+
+```scala
+val ordersStream: KStream[OrderId, Order] = discountedOrdersStream.selectKey { (_, order) => order.orderId }
+```
+
+We have to pay attention when we change the key of a stream. In fact, the broker needs to move the messages among nodes to ensure the co-partitioning constraint. This operation is called shuffling, and it's very time and resource-consuming.
+
+The `ordersStream` stream represents the same information as the `discountedOrdersStream`, but is indexed by `OrderId`. Now, we've met the preconditions to join our streams:
+
+```scala
+val paidOrders: KStream[OrderId, Order] = {
+
+ val joinOrdersAndPayments = (order: Order, payment: Payment) =>
+ if (payment.status == "PAID") Option(order) else Option.empty[Order]
+
+ val joinWindow = JoinWindows.of(Duration.of(5, ChronoUnit.MINUTES))
+
+ ordersStream.join[Payment, Option[Order]](paymentsStream)(joinOrdersAndPayments, joinWindow)
+ .flatMapValues(maybeOrder => maybeOrder.toIterable)
+}
+```
+
+The final stream, `paidOrders`, contains all the orders paid at most five minutes after their arrival into the application. As we can see, we applied a joining sliding window of five minutes:
+
+```scala
+val joinWindow = JoinWindows.of(Duration.of(5, ChronoUnit.MINUTES))
+```
+
+As the stream-stream join uses the key of the messages, we only need to provide the mapping function of the joined values, other than the joining window.
+
+To explain the semantic of the join, we can look at the following table. The first column mimics the passing of time, while the following two columns represent the values of the messages in the two starting streams. For the sake o simplicity, the key, aka the `OrderId`, is the same for all the messages. Moreover, the table represents a five minutes window, starting from the arrival of the first message:
+
+| Timestamp | `ordersStream` | `paymentsStream` | Joined value |
+|-----------|-------------------|------------------------|----------------|
+| 1 | `Order("order1")` | | |
+| 2 | | `Payment("REQUESTED")` | `Option.empty` |
+| 3 | | `Payment("ISSUED")` | `Option.empty` |
+| 4 | | `Payment("PAID")` | `Some(Order)` |
+
+
+At time 1, we receive the discounted order with id `order1`. From time 2 to 4, we get three messages into the `paymentsStream` for the same order, representing three different payment statuses. All three messages will join the order since we received them inside the defined window.
+
+The three types of join transformation we presented represent only the primary examples of the joins available in the Kafka Streams library. In fact, the library offers to developers also left join and outer joins, but their description is far beyond the scope of this article.
+
+## 7. The Kafka Streams Application
+
+Once we have defined the desired topology for our application, it's time to materialize and execute it. Materializing the topology is easy since we only have to call the `build` method on the instance of the `StreamBuilder` we have used so far:
+
+```scala
+val topology: Topology = builder.build()
+```
+
+The object of type `Topology` represents the whole set of transformations we defined. Interestingly, we can also print the topology simply calling the `describe` method on it and obtaining a `TopologyDescription` description object that is suitable for printing:
+
+```scala
+println(topology.describe())
+```
+
+The printed topology of our leading example is the following:
+
+```
+Topologies:
+ Sub-topology: 0
+ Source: KSTREAM-SOURCE-0000000000 (topics: [orders-by-user])
+ --> KSTREAM-JOIN-0000000007
+ Processor: KSTREAM-JOIN-0000000007 (stores: [discount-profiles-by-user-STATE-STORE-0000000001])
+ --> KSTREAM-LEFTJOIN-0000000008
+ <-- KSTREAM-SOURCE-0000000000
+ Processor: KSTREAM-LEFTJOIN-0000000008 (stores: [])
+ --> KSTREAM-KEY-SELECT-0000000009
+ <-- KSTREAM-JOIN-0000000007
+ Processor: KSTREAM-KEY-SELECT-0000000009 (stores: [])
+ --> KSTREAM-FILTER-0000000012
+ <-- KSTREAM-LEFTJOIN-0000000008
+ Processor: KSTREAM-FILTER-0000000012 (stores: [])
+ --> KSTREAM-SINK-0000000011
+ <-- KSTREAM-KEY-SELECT-0000000009
+ Source: KSTREAM-SOURCE-0000000002 (topics: [discount-profiles-by-user])
+ --> KTABLE-SOURCE-0000000003
+ Sink: KSTREAM-SINK-0000000011 (topic: KSTREAM-KEY-SELECT-0000000009-repartition)
+ <-- KSTREAM-FILTER-0000000012
+ Processor: KTABLE-SOURCE-0000000003 (stores: [discount-profiles-by-user-STATE-STORE-0000000001])
+ --> none
+ <-- KSTREAM-SOURCE-0000000002
+
+ Sub-topology: 1 for global store (will not generate tasks)
+ Source: KSTREAM-SOURCE-0000000005 (topics: [discounts])
+ --> KTABLE-SOURCE-0000000006
+ Processor: KTABLE-SOURCE-0000000006 (stores: [discounts-STATE-STORE-0000000004])
+ --> none
+ <-- KSTREAM-SOURCE-0000000005
+ Sub-topology: 2
+ Source: KSTREAM-SOURCE-0000000010 (topics: [payments])
+ --> KSTREAM-WINDOWED-0000000015
+ Source: KSTREAM-SOURCE-0000000013 (topics: [KSTREAM-KEY-SELECT-0000000009-repartition])
+ --> KSTREAM-WINDOWED-0000000014
+ Processor: KSTREAM-WINDOWED-0000000014 (stores: [KSTREAM-JOINTHIS-0000000016-store])
+ --> KSTREAM-JOINTHIS-0000000016
+ <-- KSTREAM-SOURCE-0000000013
+ Processor: KSTREAM-WINDOWED-0000000015 (stores: [KSTREAM-JOINOTHER-0000000017-store])
+ --> KSTREAM-JOINOTHER-0000000017
+ <-- KSTREAM-SOURCE-0000000010
+ Processor: KSTREAM-JOINOTHER-0000000017 (stores: [KSTREAM-JOINTHIS-0000000016-store])
+ --> KSTREAM-MERGE-0000000018
+ <-- KSTREAM-WINDOWED-0000000015
+ Processor: KSTREAM-JOINTHIS-0000000016 (stores: [KSTREAM-JOINOTHER-0000000017-store])
+ --> KSTREAM-MERGE-0000000018
+ <-- KSTREAM-WINDOWED-0000000014
+ Processor: KSTREAM-MERGE-0000000018 (stores: [])
+ --> KSTREAM-FLATMAPVALUES-0000000019
+ <-- KSTREAM-JOINTHIS-0000000016, KSTREAM-JOINOTHER-0000000017
+ Processor: KSTREAM-FLATMAPVALUES-0000000019 (stores: [])
+ --> KSTREAM-SINK-0000000020
+ <-- KSTREAM-MERGE-0000000018
+ Sink: KSTREAM-SINK-0000000020 (topic: paid-orders)
+ <-- KSTREAM-FLATMAPVALUES-0000000019
+```
+
+The above text representation is a bit hard to read -- fortunately, there's an open-source project which can create a visual graph of the topology, starting from the above output. The project is called [`kafka-stream-viz`](https://zz85.github.io/kafka-streams-viz/), and the generated visual graph for our topology is the following:
+
+
+
+This graphical representation allows us to follow the sequences of transformations easier than the text form. In addition, it becomes straightforward to understand which transformation is stateful and so requires a state store.
+
+Once we materialize the topology, we can effectively run the Kafka Streams application. First, we have to set the url to connect to the Kafka cluster, and the name of the application:
+
+```scala
+val props = new Properties
+props.put(StreamsConfig.APPLICATION_ID_CONFIG, "orders-application")
+props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.stringSerde.getClass)
+```
+
+In our example, we also configured the default `Serde` type for keys. Then, the last step is to create the `KafkaStream` application and start it:
+
+```scala
+val application: KafkaStreams = new KafkaStreams(topology, props)
+application.start()
+```
+
+Finally, the complete application for the leading example of our article, which leads to paid orders, is the following:
+
+```scala
+object KafkaStreamsApp {
+
+ implicit def serde[A >: Null : Decoder : Encoder]: Serde[A] = {
+ val serializer = (a: A) => a.asJson.noSpaces.getBytes
+ val deserializer = (aAsBytes: Array[Byte]) => {
+ val aAsString = new String(aAsBytes)
+ val aOrError = decode[A](aAsString)
+ aOrError match {
+ case Right(a) => Option(a)
+ case Left(error) =>
+ println(s"There was an error converting the message $aOrError, $error")
+ Option.empty
+ }
+ }
+ Serdes.fromFn[A](serializer, deserializer)
+ }
+
+ // Topics
+ final val OrdersByUserTopic = "orders-by-user"
+ final val DiscountProfilesByUserTopic = "discount-profiles-by-user"
+ final val DiscountsTopic = "discounts"
+ final val OrdersTopic = "orders"
+ final val PaymentsTopic = "payments"
+ final val PayedOrdersTopic = "paid-orders"
+
+ type UserId = String
+ type Profile = String
+ type Product = String
+ type OrderId = String
+
+ case class Order(orderId: OrderId, user: UserId, products: List[Product], amount: Double)
+
+ // Discounts profiles are a (String, String) topic
+
+ case class Discount(profile: Profile, amount: Double)
+
+ case class Payment(orderId: OrderId, status: String)
+
+ val builder = new StreamsBuilder
+
+ val usersOrdersStreams: KStream[UserId, Order] = builder.stream[UserId, Order](OrdersByUserTopic)
+
+ def paidOrdersTopology(): Unit = {
+ val userProfilesTable: KTable[UserId, Profile] =
+ builder.table[UserId, Profile](DiscountProfilesByUserTopic)
+
+ val discountProfilesGTable: GlobalKTable[Profile, Discount] =
+ builder.globalTable[Profile, Discount](DiscountsTopic)
+
+ val ordersWithUserProfileStream: KStream[UserId, (Order, Profile)] =
+ usersOrdersStreams.join[Profile, (Order, Profile)](userProfilesTable) { (order, profile) =>
+ (order, profile)
+ }
+
+ val discountedOrdersStream: KStream[UserId, Order] =
+ ordersWithUserProfileStream.join[Profile, Discount, Order](discountProfilesGTable)(
+ { case (_, (_, profile)) => profile }, // Joining key
+ { case ((order, _), discount) => order.copy(amount = order.amount * discount.amount) }
+ )
+
+ val ordersStream: KStream[OrderId, Order] = discountedOrdersStream.selectKey { (_, order) => order.orderId }
+
+ val paymentsStream: KStream[OrderId, Payment] = builder.stream[OrderId, Payment](PaymentsTopic)
+
+ val paidOrders: KStream[OrderId, Order] = {
+
+ val joinOrdersAndPayments = (order: Order, payment: Payment) =>
+ if (payment.status == "PAID") Option(order) else Option.empty[Order]
+
+ val joinWindow = JoinWindows.of(Duration.of(5, ChronoUnit.MINUTES))
+
+ ordersStream.join[Payment, Option[Order]](paymentsStream)(joinOrdersAndPayments, joinWindow)
+ .flatMapValues(maybeOrder => maybeOrder.toIterable)
+ }
+
+ paidOrders.to(PayedOrdersTopic)
+ }
+
+ def main(args: Array[String]): Unit = {
+ val props = new Properties
+ props.put(StreamsConfig.APPLICATION_ID_CONFIG, "orders-application")
+ props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+ props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.stringSerde.getClass)
+
+ paidOrdersTopology()
+
+ val topology: Topology = builder.build()
+
+ println(topology.describe())
+
+ val application: KafkaStreams = new KafkaStreams(topology, props)
+ application.start()
+ }
+}
+```
+
+And, that's all about the Kafka Streams library, folks!
+
+## 8. Let's Run It
+
+At this point, we defined a complete working Kafka application, which uses many transformations and joins. Now, it's time to test the developed topology, sending messages to the various Kafka topics.
+
+First, let's fill the tables, starting from the topic `discounts`. We have to send some messages associating a profile with a discount:
+
+```shell
+kafka-console-producer \
+ --topic discounts \
+ --broker-list localhost:9092 \
+ --property parse.key=true \
+ --property key.separator=,
+
+profile1,{"profile":"profile1","amount":0.5 }
+profile2,{"profile":"profile2","amount":0.25 }
+profile3,{"profile":"profile3","amount":0.15 }
+```
+
+We created three profiles in the above command with a discount of 50%, 25%, and 15%, respectively.
+
+The next step is to create some users and associated them with a discount profile in the topic `discount-profiles-by-user`:
+
+```shell
+kafka-console-producer \
+ --topic discount-profiles-by-user \
+ --broker-list localhost:9092 \
+ --property parse.key=true \
+ --property key.separator=,
+
+Daniel,profile1
+Riccardo,profile2
+```
+
+We are ready to insert our first order into the system, using the topic `orders-by-user`. As the name of the topic said, the keys of the following messages are user-ids:
+
+```shell
+kafka-console-producer \
+ --topic orders-by-user \
+ --broker-list localhost:9092 \
+ --property parse.key=true \
+ --property key.separator=,
+
+Daniel,{"orderId":"order1","user":"Daniel","products":[ "iPhone 13","MacBook Pro 15"],"amount":4000.0 }
+Riccardo,{"orderId":"order2","user":"Riccardo","products":["iPhone 11"],"amount":800.0}
+```
+
+Now, we must pay the above orders. So, we send the messages representing the payment transaction. The topic storing such messages is called `payments`:
+
+```shell
+kafka-console-producer \
+ --topic payments \
+ --broker-list localhost:9092 \
+ --property parse.key=true \
+ --property key.separator=,
+
+order1,{"orderId":"order1","status":"PAID"}
+order2,{"orderId":"order2","status":"PENDING"}
+```
+
+If everything goes right, into the topic `paid-orders` we should find a message containing the paid order of the user `"Daniel"`, containing an `"iPhone 13"` and a `"MacBook Pro 15"`, and worth 2,000.0 Euro. We can read the messages of the topic using the `kafka-console-consumer.sh` shell command:
+
+```shell
+kafka-console-consumer \
+ --bootstrap-server localhost:9092 \
+ --topic paid-orders \
+ --from-beginning
+```
+
+The above command will read the following message, concluding our journey in the Kafka Streams library:
+
+```
+{"orderId":"order1","user":"Daniel","products":["iPhone 13","MacBook Pro 15"],"amount":2000.0}
+```
+
+## 9. Conclusions
+
+This article introduced the Kafka Streams library, a Kafka client library based on top of the Kafka consumers and producers API. In detail, we focused on the Stream DSL part of the library, which lets us represent the stream's topology at a higher level of abstraction. After introducing the basic building blocks of the DSL, `KStream`, `KTable`, and `GlobalKTable`, we showed the primary operations defined on them, both the stateless and the stateful ones. Then, we talked about joins, one of the most relevant features of Kafka Streams. Finally, we wired all together, and we learned how to start a Kafka Streams application.
+
+The Kafka Streams library is vast, and it offers many more features than we saw. For example, we've not talked about the Processor API and how it's possible to query a state store directly. However, the given information should be sufficient to have a solid base to learn the advanced feature of the excellent and helpful library.
diff --git a/_posts/2021-11-21-akka-streams-backpressure.md b/_posts/2021-11-21-akka-streams-backpressure.md
new file mode 100644
index 000000000000..5c9904b68272
--- /dev/null
+++ b/_posts/2021-11-21-akka-streams-backpressure.md
@@ -0,0 +1,456 @@
+---
+title: "Akka Streams Backpressure"
+date: 2021-11-21
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka, akka streams]
+excerpt: "Akka Streams implements backpressure, a critical component of the Reactive Streams specification. This article is a demonstration of this mechanism."
+---
+
+This article is a long-overdue written analogue to [this video](https://www.youtube.com/watch?v=L5FAyCCWGL0), which discusses one of the most important aspects of a reactive system using Akka Streams.
+
+## 1. Background
+
+What's Akka Streams again?
+
+It's this library that allows us to write systems in which data is received in pieces as a stream, instead of all at once. Akka actors are a powerful tool which we can use to implement any kind of business logic, but the particular use case of data transfer in a continuous, stable, fast, resilient and fault-tolerant manner is too complex to be implemented over and over on top of actors.
+
+Akka Streams implement (implements?) the [Reactive Streams](https://www.reactive-streams.org/) specification, which describes how a reactive system is supposed to work and what the main concepts and abstractions are.
+
+We discuss this in detail in the [Akka Streams](https://rockthejvm.com/p/akka-streams/) course, but to recap really quickly:
+
+- A reactive system is built out of components which have certain roles. We have _sources_ (emitters of elements), _sinks_ (receivers of elements), _flows_ (transformers of elements), among other kinds. We build a system by connecting these components together in a graph.
+- Akka Streams components work with a _demand-based_ protocol. In other words, data flows through the graph as a response to demand from receivers. Producers then comply and send more elements downstream.
+- A second (transparent) protocol kicks in when production of elements is faster than demand. This protocol (backpressure) slows down the producers and ensures no data is lost.
+
+## 2. Getting Started
+
+Let's add the Akka Streams library to the `build.sbt` file of our project:
+
+```scala
+val akkaVersion = "2.6.13"
+
+libraryDependencies += Seq(
+ // among (perhaps) your other libraries)
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion
+)
+```
+
+After that, we'll spin up a simple application with Akka Streams by creating an actor system:
+
+```scala
+object AkkaStreamsBackpressure {
+ implicit val system: ActorSystem[_] = ActorSystem(Behaviors.empty, "StreamsSystem")
+
+ // rest of the code to follow
+}
+```
+
+We create the actor system as `implicit` because it also contains the tools to allocate the resources for the Akka Streams components we're going to add and run. When we actually start the stream, the actor system will be passed automatically.
+
+If you haven't worked with Akka Streams components yet, here's how we can build a few components:
+
+```scala
+val source = Source(1 to 1000)
+val flow = Flow[Int].map(_ * 10)
+val sink = Sink.foreach[Int](println)
+```
+
+and we can connect these components into a graph
+
+```scala
+val graph = source.via(flow).to(sink)
+```
+
+however, this graph is just a blueprint for a running computation, which we can start by calling the `run` method on it.
+
+In order to demonstrate backpressure, we're going to create some slightly different components so that we can investigate the difference in behavior:
+
+```scala
+val slowSink = Sink.foreach[Int] { x =>
+ Thread.sleep(1000)
+ println(x)
+}
+
+val debuggingFlow = Flow[Int].map { x =>
+ println(s"[flow] ${x}")
+ x
+}
+```
+
+## 3. No Backpressure Yet
+
+After learning the concepts and the basic components of Akka Streams, a naive approach would be to just create a slow consumer (like the above sink) and connect a stream, like so:
+
+```scala
+def demoNoBackpressure(): Unit = {
+ source.via(debuggingFlow).to(slowSink).run()
+}
+```
+
+What we see in the console is:
+
+```text
+[flow] 1
+1
+[flow] 2
+2
+[flow] 3
+3
+[flow] 4
+4
+[flow] 5
+5
+[flow] 6
+6
+[flow] 7
+7
+```
+
+and each pair is printed once per second (as the bottleneck is the flow). Is this backpressure? In effect, our flow is slowed down, isn't it?
+
+The answer might be surprising: this is _not_ an example of backpressure. When we connect Akka Streams components in this way, i.e. `source.via(debuggingFlow).to(slowSink)`, the Akka Streams library will make an interesting assumption.
+
+* Because we'd like these components to be as fast as possible in real life, we assume that each transformation along the entire graph is very quick.
+* Because components run on top of actors, sending elements between subsequent components is based on message exchanges between actors.
+* Because the components are considered to be fast, message exchanges are assumed to be a significant _overhead_: the time for a message to be sent, enqueued and received is (in this assumption) comparable with the time it takes for the data to be processed.
+
+For these reasons, Akka Streams automatically _fuses_ components together: if we connect components with the `via` and `to` methods, Akka Streams will actually run them on _the same actor_ to eliminate these message exchanges. The direct consequence is that all data processing happens _sequentially_.
+
+But wait, isn't Akka Streams supposed to parallelize everything, because it's based on actors?
+
+This may or may not be what you want. Akka Streams is a very general library, and for the use-cases described above (components with fast processing times), it's actually better to run them on the same actor.
+
+## 4. The Beginning of Backpressure
+
+The assumption we described earlier breaks down when the data processing is slow for one (or more) of the components involved. Because everything happens sequentially, this component will slow down the entire stream. For this situation, we need _async boundaries_: a way to specify which part(s) of the stream will run on one actor, which part(s) on another actor, etc. This simple method will suffice:
+
+```scala
+def demoBackpressure(): Unit = {
+ source.via(debuggingFlow).async.to(slowSink).run()
+}
+```
+
+The `async` is the crux here: everything to the left of `async` runs on one actor, everything on the right runs on another actor. Obviously, we can add many `async` calls, even after each component if we want to. In this case, the source and flow run on the same actor (call this actor 1), and the slow sink will run on a different actor (say actor 2).
+
+If we run this method instead, we get a (perhaps surprisingly) different output. My notes are prefixed with `----` in the output below:
+
+```text
+---- burst, all at once
+[flow] 1
+[flow] 2
+[flow] 3
+[flow] 4
+[flow] 5
+[flow] 6
+[flow] 7
+[flow] 8
+[flow] 9
+[flow] 10
+[flow] 11
+[flow] 12
+[flow] 13
+[flow] 14
+[flow] 15
+[flow] 16
+---- then slow, one per second
+1
+2
+3
+4
+5
+6
+7
+---- burst again
+[flow] 17
+[flow] 18
+[flow] 19
+[flow] 20
+[flow] 21
+[flow] 22
+[flow] 23
+[flow] 24
+---- slow again, one per second
+8
+9
+10
+11
+12
+13
+14
+15
+---- burst
+[flow] 25
+[flow] 26
+[flow] 27
+[flow] 28
+[flow] 29
+[flow] 30
+[flow] 31
+[flow] 32
+---- etc.
+16
+17
+```
+
+Why do we get such a different behavior?
+
+Because the source + flow combo and the sink run on different actors, now we have message exchanges between the components of the stream, so we have the demand/backpressure protocol we described at the beginning of the article.
+
+Here's a breakdown of what's happening in this case:
+
+1. The sink demands an element, which starts the flow + source.
+2. In a snap, the sink receives an element, but it takes 1 second to process it, so it will send a _backpressure signal_ upstream.
+3. During that time, the flow will attempt to keep the throughput of the source, and _buffer 16 elements_ internally.
+4. Once the flow's buffer is full, it will stop receiving new elements.
+5. Once per second, the sink will continue to receive an element and print it to the console (the second, slow batch).
+6. After 8 elements, the flow's buffer becomes half-empty. It will then resume the source and print 8 more elements in a burst, until its buffer is full again.
+7. The slow sink will keep doing its thing.
+8. After 8 more elements, the flow's buffer becomes half-empty again, which will resume the source.
+9. And so on.
+
+As you can see, the natural response to a backpressure signal is to attempt _buffering new elements locally_ in order to maintain the producer throughput for as long as possible. Each Akka Streams component has its own internal buffer (by default 16 elements).
+
+## 5. Customizing Buffering
+
+We can control what happens when a consumer is slow. Akka Streams components allow us to
+
+* buffer elements locally, with a configurable size
+* drop data to maintain throughput, with configurable deletion strategies
+* send backpressure signal upstream
+* fail and tear down the stream altogether
+
+We can decorate sources and flows with a configurable buffer which has all the capabilities described above. A simple example would look like
+
+```scala
+def demoBackpressure(): Unit = {
+ source.via(debuggingFlow.buffer(10, OverflowStrategy.backpressure)).async.to(slowSink).run()
+}
+```
+
+where `buffer(10, OverflowStrategy.backpressure)` means
+
+* an additional buffer of 10 elements
+* if the buffer is full, the decision will be to send a backpressure signal upstream to slow down the source
+
+Some demonstrations follow.
+
+### 5.1. Backpressure - Slow Down the Stream
+
+The stream is exactly the one in the example above:
+
+```scala
+source.via(debuggingFlow.buffer(10, OverflowStrategy.backpressure)).async.to(slowSink).run()
+```
+
+The output is (the `----` are my notes)
+
+```text
+---- burst
+[flow] 1
+[flow] 2
+[flow] 3
+[flow] 4
+[flow] 5
+[flow] 6
+[flow] 7
+[flow] 8
+[flow] 9
+[flow] 10
+[flow] 11
+[flow] 12
+[flow] 13
+[flow] 14
+[flow] 15
+[flow] 16
+[flow] 17
+[flow] 18
+[flow] 19
+[flow] 20
+[flow] 21
+[flow] 22
+[flow] 23
+[flow] 24
+[flow] 25
+[flow] 26
+---- slow
+1
+2
+3
+4
+5
+6
+7
+---- burst
+[flow] 27
+[flow] 28
+[flow] 29
+[flow] 30
+[flow] 31
+[flow] 32
+[flow] 33
+[flow] 34
+---- slow
+8
+9
+10
+11
+12
+13
+14
+15
+[flow] 35
+[flow] 36
+[flow] 37
+[flow] 38
+[flow] 39
+[flow] 40
+[flow] 41
+[flow] 42
+---- and so on
+16
+```
+
+We now have 26 items being printed in the first burst because we have 16 elements from the original flow, plus 10 of the additional buffer. Otherwise the behavior is identical to the one described earlier.
+
+### 5.2. Dropping Data - Oldest (Head)
+
+If we absolutely need to maintain throughput, we have no choice but to start dropping data. One strategy is to make room for the incoming element by removing the _oldest_ element in the current buffer (the head of the list).
+
+```scala
+source.via(debuggingFlow.buffer(10, OverflowStrategy.dropHead)).async.to(slowSink).run()
+```
+
+The output is:
+
+```text
+---- burst
+[flow] 1
+[flow] 2
+[flow] 3
+---- omitting for brevity
+[flow] 1000
+---- slow
+991
+992
+993
+994
+995
+996
+997
+998
+999
+1000
+---- end of stream
+```
+
+Here, the behavior is different: the flow is fast, but it has to start dropping data, and with every incoming element, the oldest current element in the buffer will be removed. So at every point, the buffer will keep the latest data, which (after 1 second) ends up being printed in the sink, slowly.
+
+### 5.3. Dropping Data - Newest (Tail)
+
+Still in the realm of dropping data, we can make room for the incoming element by removing the latest element in the buffer. The code looks like this:
+
+```scala
+source.via(debuggingFlow.buffer(10, OverflowStrategy.dropTail)).async.to(slowSink).run()
+```
+
+And the output is:
+
+```text
+---- burst
+[flow] 1
+[flow] 2
+[flow] 3
+---- omitting for brevity
+[flow] 1000
+---- slow
+1
+2
+3
+4
+5
+6
+7
+8
+9
+1000
+```
+
+In this case, every incoming element displaces the previous newest element in the buffer. At the end (after 1 second), the buffer will contain the 9 oldest elements (1 through 9) and the absolute newest element, 1000.
+
+### 5.3. Dropping Data - New Element
+
+If the buffer overflows, we can also choose to keep the buffer as it is and drop the incoming element instead, because we may consider the historical data more important. The difference is in using `dropNew`:
+
+```scala
+source.via(debuggingFlow.buffer(10, OverflowStrategy.dropNew)).async.to(slowSink).run()
+```
+
+And the output is:
+
+```text
+---- burst
+[flow] 1
+[flow] 2
+[flow] 3
+---- omitting for brevity
+[flow] 1000
+---- slow
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+```
+
+In this case, the buffer is intact by the time the sink starts printing, so only the numbers 1-10 get shown.
+
+### 5.4. Dropping Data - Entire Buffer
+
+Finally, we can decide to remove the entire buffer if it overflows, and start fresh. The overflow strategy is called `dropBuffer`:
+
+```scala
+source.via(debuggingFlow.buffer(10, OverflowStrategy.dropNew)).async.to(slowSink).run()
+```
+
+With a surprising output:
+
+```text
+---- burst
+[flow] 1
+[flow] 2
+[flow] 3
+---- omitting for brevity
+[flow] 1000
+---- slow
+991
+992
+993
+994
+995
+996
+997
+998
+999
+1000
+---- end of stream
+```
+
+Same as with `dropHead`, but here the mechanism is different: the buffer is removed entirely with each overflow, so in the end we'll have the last 10 elements. If instead of a source 1000 elements we used 1001 elements, we would have had yet another buffer drop with the last element, which would have been the only one being printed.
+
+### 5.5. Teardown
+
+This last resort is uninspiring: when we want both the throughput to be high and the data to be intact, the only thing we can do in case of a buffer overflow is to throw an exception, which will fail the entire stream. The strategy is called `fail`.
+
+## 6. Conclusion
+
+In this article, we went through backpressure in reactive systems, how Akka Streams manages it, and we did some demos on what should happen in case we want to maintain throughput/lose data or slow down stream/keep data.
+
+The video version can be found here:
+
+{% include video id="L5FAyCCWGL0" provider="youtube" %}
diff --git a/_posts/2021-12-04-scala-modes-of-evaluation.md b/_posts/2021-12-04-scala-modes-of-evaluation.md
new file mode 100644
index 000000000000..fa077b7b8d0e
--- /dev/null
+++ b/_posts/2021-12-04-scala-modes-of-evaluation.md
@@ -0,0 +1,78 @@
+---
+title: "Evaluation Modes in Scala"
+date: 2021-12-04
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, philosophical]
+excerpt: "We'll take a look at some core Scala constructs and look at them from a different angle than we're used to."
+---
+
+This article is a bit shorter than usual, but I hope it will share a different kind of insight. We're not going to explore new features, external libraries, create apps or demos. We're going to take a look at some core Scala constructs and _understand_ them in a different context.
+
+## 1. The 3 Evaluation Modes
+
+When we want to compute a value, we can think of two different aspects to how that value is computed. There are more, but I will focus on the following:
+
+1. The time. We can choose to compute a value _now_ vs compute a value _later_, i.e. we declare that value now, but the evaluation itself happens at a later point, _when that value is needed_.
+2. The memory. We can choose to _store_ that value in memory, or we can choose not to, which means that we'd have to _recompute_ that value whenever we need it.
+
+The time aspect divides computations into now vs later. The memory aspect divides computations into _"memoized"_ and _non-memoized_. I put "memoized" in quotes, because memoization is considered to be an optimization technique. However, at a foundational level, storing vs. not storing is a fundamental aspect of any computation. Note that there are many more aspects of computation that I did not refer to, e.g. asynchrony, side effects, etc.
+
+In this classification, these two orthogonal aspects give us 4 kinds of computations:
+
+- computed now, memoized
+- computed later, memoized
+- computed now, non-memoized
+- computed later, non-memoized
+
+Arguably, type 3 isn't useful: computing a value immediately (i.e. at the point of definition) without storing the value is pure waste. So I'll consider the last type (computed later, non-memoized) as "type 3".
+
+## 2. Scala Constructs For The 4 Evaluation Modes
+
+How does Scala implement this classification? Here's a quick breakdown.
+
+Type 1: How do we define a value that we compute _right now_ and store it in memory? By defining a `val`:
+
+```scala
+val meaningOfLife = 40 + 2
+```
+
+When we define a `val`, it will _always_ be computed at the point of definition. The expression is evaluated and the result is stored.
+
+Type 2: How do we define a value that's computed _later_, but stored when computed? Scala has a construct called a `lazy val`:
+
+```scala
+lazy val complexThing = (1 to 42).map(_ => 1).reduce(_ + _)
+```
+
+This value is defined, but the expression will only be evaluated _at the point of use_. Once evaluated, it's stored, so we can reference (and reuse) that value every time after that. Note that the question of "computed later" does not refer to _asynchrony_. When we define a Future, for example, the Future itself is available (as an instance of a type), and it's its internal mechanism that decides to use threads, fetch results asynchronously, etc.
+
+Type 3: How do we define a value that's computed later, but not stored, i.e. if we want to use it again, we'll have to recompute it? It's a `def`:
+
+```scala
+def recomputed = 3 + 39
+```
+
+This is potentially the most surprising. But it's true: every time we use `recomputed`, the expression is evaluated again!
+
+So we have the following associations:
+
+- computed now, memoized: `val`
+- computed later, memoized: `lazy val`
+- computed later, non-memoized: `def`
+
+But since our programs are not built just out of plain values, but also functions taking arguments, we can also transfer this small classification to arguments as well:
+
+- computed now, memoized: plain argument
+- computed later, non-memoized: by-name argument
+- computed later, memoized: by name argument + lazy val (call by need)
+
+## 3. Conclusion: The Philosophy Hidden in Plain Sight
+
+Scala is brilliant on so many levels, but this particular aspect of Scala makes it so powerful, because it leverages concepts we (as programmers) are familiar with in different contexts, and gave them new meanings. We think of
+
+- `val`s as constants
+- `lazy val`s as constants computed later
+- `def`s as methods
+
+but how many times do think about what computations expressed in these terms _are_?
diff --git a/_posts/2021-12-20-tagless-final.md b/_posts/2021-12-20-tagless-final.md
new file mode 100644
index 000000000000..32b4ca61ae33
--- /dev/null
+++ b/_posts/2021-12-20-tagless-final.md
@@ -0,0 +1,372 @@
+---
+title: "Tagless Final in Scala"
+date: 2021-12-20
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, abstract]
+excerpt: "Demystifying the tagless final pattern in Scala. TLDR: it's got nothing to do with type classes."
+---
+
+This article is about a popular topic in the Scala world, which is a very weird combination, probably the weirdest I've come across on the blog.
+
+- It's (described to be) highly abstract and solves a very general kind of problem.
+- It caused a giant amount of confusion in the Scala community, including myself for a very long time.
+- It's poorly covered in articles, books and videos. I've read and watched everything I could get my hands on and still had gaps in my understanding.
+- It's very popular and widely used, especially in code based on the Cats Effect/Typelevel stack.
+
+That final point is striking. It seems as though TF is used not _because of_, but _in spite of_ its structure. Some argued it causes more problems than it solves. Debates started. Competing libraries emerged.
+
+I'd argue TF's bad rap (if there is one) could be solved with a better learning curve. This article is the summary of everything I know so far, in what I hope to be an easily digestible form.
+
+There's also a video form, which you can watch here:
+
+{% include video id="m3Qh-MmWpbM" provider="youtube" %}
+
+Special thanks to
+
+- the original [TF paper](https://okmij.org/ftp/tagless-final/JFP.pdf) formalizing the pattern with examples in Haskell
+- pretty much every other article, book or video referencing TF
+
+## 1. The Expression Problem
+
+Because we write FP, we think in terms of expressions. In fact, a purely functional program is nothing else but a single expression (if quite big) that computes a value (albeit quite complex, e.g. a server or a data pipeline). This is what I teach in both my [Scala](https://rockthejvm.com/p/scala) [courses](https://rockthejvm.com/p/advanced-scala) here at Rock the JVM.
+
+For pure FP to work properly in a strongly typed language like Scala, we immediately get into the _expression problem_. Namely, if we're given an expression computing a value, we would like to be able to evaluate it and return the proper value of the right _type_ for that expression.
+
+Let's imagine something simple. We want to write a program that is able to evaluate boolean expressions. To do so, we can encode the boolean operands and operators as nodes in a binary tree. An expression is a tree consisting of operands (leaves) and operators (branches), and we can evaluate the expression by traversing the tree and collapsing its result to a single value. For instance, the expression
+
+```scala3
+true && false || true
+```
+
+can be modeled as
+
+```scala3
+Or(And(Leaf(true), Leaf(false)), Leaf(true))
+```
+
+considering we had data structures to represent each node in the tree. Naming `Leaf` as `B` (easier), we can immediately build up a few case classes to represent our particular expression problem:
+
+```scala3
+trait Expr // the "tree"
+case class B(boolean: Boolean) extends Expr
+case class Or(left: Expr, right: Expr) extends Expr
+case class And(left: Expr, right: Expr) extends Expr
+case class Not(expr: Expr) extends Expr
+```
+
+and an evaluation function would look like this:
+
+```scala3
+def eval(expr: Expr): Boolean = expr match {
+ case B(b) => b
+ case Or(a, b) => eval(a) || eval(b)
+ case And(a, b) => eval(a) && eval(b)
+ case Not(e) => !eval(e)
+}
+```
+
+Of course, the computer can already calculate boolean expressions of any complexity, but bear with me. Assume that we'd now like to enhance our evaluation capabilities to now include integers as well, so we'd like to have an additional set of case classes
+
+```scala3
+case class I(int: Int) extends Expr
+case class Sum(left: Expr, right: Expr) extends Expr
+```
+
+but now evaluating an expression is not so straightforward, because not only do we have to add some additional cases in our pattern match, but also lose type safety and type-cast everything:
+
+```scala3
+def eval_v2(expr: Expr): Any = expr match { // notice the Any return type
+ case B(b) => b
+ case Or(a, b) => eval(a).asInstanceOf[Boolean] || eval(b).asInstanceOf[Boolean]
+ // casts everywhere
+}
+```
+
+not to mention that some of those expressions can have the incorrect type, e.g. `Or(I(1), B(true))` which will crash when type-casting. So the problem becomes: how do we return the right type for the right expression?
+
+The expression problem is very general. The example we have here might look contrived, but we'll see a more realistic example later.
+
+## 2. First Solution: Tagging
+
+The expression problem reduces to differentiating numerical expressions from boolean expressions without destroying the code quality, losing type safety or using type casts everywhere.
+
+One easy way of doing this differentiation is by using some additional data inside each instance of `Expr` to be able to tell whether we should be using type casts, and what type to expect:
+
+```scala3
+trait Expr(val tag: String)
+case class B(boolean: Boolean) extends Expr("bool")
+case class Or(left: Expr, right: Expr) extends Expr("bool")
+case class And(left: Expr, right: Expr) extends Expr("bool")
+case class Not(expr: Expr) extends Expr("bool")
+case class I(int: Int) extends Expr("int")
+case class Sum(left: Expr, right: Expr) extends Expr("int")
+
+def eval(expr: Expr): Any = expr match {
+ case B(b) => b
+ case Or(left, right) =>
+ if (left.tag == "bool" && right.tag == "bool")
+ eval(left).asInstanceOf[Boolean] || eval(right).asInstanceOf[Boolean]
+ else
+ throw new IllegalArgumentException("attempting to evaluate an expression with improperly typed operands")
+ // same for others
+}
+```
+
+This approach has the benefit of doing "type" checks (by checking the tag) and returning the correct result for each expression. However, the type check still happens at runtime, we still don't have true type checking (by the `Any` return type) and we're still doing type casts. A slight improvement would be to move the "type" checks at the construction phase of each data structure, e.g.
+
+```scala3
+case class Or(left: Expr, right: Expr) extends Expr("bool") {
+ assert(left.tag == "bool" || right.tag == "bool")
+}
+```
+
+but this would still crash at runtime. We'd like something better
+
+## 3. Removing Tags
+
+Why add tags and check them at runtime, when we have a strongly typed language that can do the type checks for us at compile time?
+
+Because the tags in the previous solution essentially added type information to the runtime, we can remove the tags and let the compiler do the type-checking automatically.
+
+```scala3
+trait Expr[A]
+case class B(boolean: Boolean) extends Expr[Boolean]
+case class Or(left: Expr[Boolean], right: Expr[Boolean]) extends Expr[Boolean]
+case class And(left: Expr[Boolean], right: Expr[Boolean]) extends Expr[Boolean]
+case class Not(expr: Expr[Boolean]) extends Expr[Boolean]
+case class I(int: Int) extends Expr[Int]
+case class Sum(left: Expr[Int], right: Expr[Int]) extends Expr[Int]
+```
+
+We got rid of the tags and added a generic type argument, which the compiler will use to check correctness. For instance, we can easily build an expression such as `Or(B(true), B(false))` but we can't build an expression such as `Or(I(1), B(true))` or `I(false)` or `B(45)`.
+
+Now, we can also make our evaluation function correctly typed:
+
+```scala3
+def eval[A](expr: Expr[A]): A = expr match {
+ case B(b) => b
+ case I(i) => i
+ case Or(left, right) => eval(left) || eval(right)
+ case Sum(left, right) => eval(left) + eval(right)
+ // etc
+}
+```
+
+Code looks cleaner, correctness is easier to prove, no more type tags, maintaining type safety. So good.
+
+This is a tagl*less* solution, because we've removed tags. It's called _tagless initial_, because we work with intermediate data structures, not with the values we care about. That would be _tagless final_, coming next. In the meantime, if we test what we have,
+
+```scala3
+def demoTagless(): Unit = {
+ import TaglessInitial._
+ println(eval(Or(B(true), And(B(true), B(false)))))
+ println(eval(Sum(I(24), I(-3))))
+}
+```
+
+it prints exactly what we're expecting: the boolean `true` and the integer `21`. As mentioned earlier, only properly-formed expressions will work, because otherwise the compiler will catch the type mismatches.
+
+## 4. Tagless Final
+
+There is another step where we can take this. Not only can we remove tags, but we can also immediately represent these expressions in terms of the evaluated value we care about (the final value). This is tagless _final_. We'll represent our expression types a bit differently,
+
+```scala3
+trait Expr[A] {
+ val value: A // the final value we care about
+}
+
+def b(boolean: Boolean): Expr[Boolean] = new Expr[Boolean] {
+ val value = boolean
+}
+
+def i(int: Int): Expr[Int] = new Expr[Int] {
+ val value = int
+}
+
+def or(left: Expr[Boolean], right: Expr[Boolean]) = new Expr[Boolean] {
+ val value = left.value || right.value
+}
+
+def and(left: Expr[Boolean], right: Expr[Boolean]) = new Expr[Boolean] {
+ val value = left.value && right.value
+}
+
+def sum(left: Expr[Int], right: Expr[Int]) = new Expr[Int] {
+ val value = left.value + right.value
+}
+
+def eval[A](expr: Expr[A]): A = expr.value
+```
+
+where our `Expr[A]` has the evaluated value directly in the instance, as a member. Each construction of another `Expr` of the right type already has the final value embedded there. Therefore, our evaluation function is almost empty, because all we need to do is just return the value embedded in the expression being passed as argument.
+
+Of course, a demonstration of this code will also print what the initial tagless solution did:
+
+```scala3
+def demoTaglessFinal(): Unit = {
+ import TaglessFinal._
+ println(eval(or(b(true), and(b(true), b(false)))))
+ println(eval(sum(i(24), i(-3))))
+}
+```
+
+This is, of course, a mere refactoring of the code in tagless initial so that we can immediately work with the final representation of our results.
+
+But that, friends, is it. This is tagless final. We started with the initial expression problem, and we solved it with this style of organizing code. I want you to think about TF as a "design pattern", because that's what we did: we designed and structured our code to fit a particular use-case.
+
+_fine print: Tagless final in the original paper is much more than a design pattern, it's a way of creating new "languages", e.g. means of computation + syntaxes, on top of existing languages, e.g. Scala. That said, for the practicality of Scala programmers, this article is focused on our pragmatic need to write good code._
+
+Now, after "this is tagless final", the big question: _where's the `F[_]`_?
+
+For context, the `F[_]` is seen very often in code bases that run Cats Effect/Typelevel libraries under the hood, where method definitions use higher-kinded types for which we require the presence of a particular type class:
+
+```scala3
+def myService[F[_]: Concurrent](...)
+```
+
+and this is often called "tagless final". However, programming against type classes and the tagless final pattern have nothing to do with each other. The only overlap is in how the code ends up looking like and the functionality restriction:
+
+- type classes are a sets of functionalities which you want to offer to some types and not for others
+- tagless final wants to prove correctness of expressions of some types and not for others
+
+See the overlap?
+
+## 5. A "Tagless Final" Refactor
+
+We can also take our own solution from the previous section and refactor it to use higher kinds.
+
+We can group all our functionalities, i.e. the ability to construct expressions, operands and operators in a single type class, implemented in terms of an abstract type `E`:
+
+```scala3
+trait Algebra[E[_]] {
+ def b(boolean: Boolean): E[Boolean]
+ def i(int: Int): E[Int]
+ def or(left: E[Boolean], right: E[Boolean]): E[Boolean]
+ def and(left: E[Boolean], right: E[Boolean]): E[Boolean]
+ def sum(left: E[Int], right: E[Int]): E[Int]
+}
+```
+
+Although "algebra" is too fancy a term for my taste and for the pragmatic Scala programmer, we'll consider it here. Given this interface, we can now imagine some concrete representations of it; one simple example below:
+
+```scala3
+case class SimpleExpr[A](value: A)
+given simpleExprAlg: Algebra[SimpleExpr] with {
+ override def b(boolean: Boolean) = SimpleExpr(boolean)
+ override def i(int: Int) = SimpleExpr(int)
+ override def or(left: SimpleExpr[Boolean], right: SimpleExpr[Boolean]) = SimpleExpr(left.value || right.value)
+ override def and(left: SimpleExpr[Boolean], right: SimpleExpr[Boolean]) = SimpleExpr(left.value && right.value)
+ override def sum(left: SimpleExpr[Int], right: SimpleExpr[Int]) = SimpleExpr(left.value + right.value)
+}
+```
+
+where the implementation of the `Algebra` for `SimpleExpr` was made a `given` (or an `implicit val` in Scala 2). The implementation is called an _interpreter_, which is one of many possible.
+
+Now, if we want to reproduce the same expressions as last time, we'll need to build _programs_, i.e. build expressions using this `Algebra` trait.
+
+```scala3
+def program1[E[_]](using alg: Algebra[E]): E[Boolean] = {
+ import alg._
+ or(b(true), and(b(true), b(false)))
+}
+
+def program2[E[_]](using alg: Algebra[E]): E[Int] = {
+ import alg._
+ sum(i(24), i(-3))
+}
+```
+
+And if we want to print the same result:
+
+```scala3
+def demoFinalTagless_v2(): Unit = {
+ import TaglessFinal_V2._
+ println(program1[SimpleExpr].value)
+ println(program2[SimpleExpr].value)
+}
+```
+
+This is what we end up with in libraries such as Cats Effect. We write very general code in terms of an effect type, and finally we plug it in at the "end of the world", where the type class instances for that effect type are brought into scope. I show that in the [Cats Effect course](https://rockthejvm.com/p/cats-effect), by the way.
+
+However, using this style is not a requirement for a solution to be "tagless final".
+
+Another question: why is there so much fuss about tagless final? It seems very abstract. How is this related to "real-life code"?
+
+Tagless final indeed is a topic for abstract algebra, with examples in Haskell (a much more mathematical language than Scala). However, the problem that tagless final solves is very practical and very general. Just to give a 30-second refactor, we have our tagless-final-with-type-classes solution below:
+
+```scala3
+object TaglessFinal_V2 {
+ trait Algebra[E[_]] {
+ def b(boolean: Boolean): E[Boolean]
+ def i(int: Int): E[Int]
+ def or(left: E[Boolean], right: E[Boolean]): E[Boolean]
+ def and(left: E[Boolean], right: E[Boolean]): E[Boolean]
+ def sum(left: E[Int], right: E[Int]): E[Int]
+ }
+
+ case class SimpleExpr[A](value: A)
+ given simpleExprAlg: Algebra[SimpleExpr] with {
+ override def b(boolean: Boolean) = SimpleExpr(boolean)
+ override def i(int: Int) = SimpleExpr(int)
+ override def or(left: SimpleExpr[Boolean], right: SimpleExpr[Boolean]) = SimpleExpr(left.value || right.value)
+ override def and(left: SimpleExpr[Boolean], right: SimpleExpr[Boolean]) = SimpleExpr(left.value && right.value)
+ override def sum(left: SimpleExpr[Int], right: SimpleExpr[Int]) = SimpleExpr(left.value + right.value)
+ }
+
+ def program1[E[_]](using alg: Algebra[E]): E[Boolean] = {
+ import alg._
+ or(b(true), and(b(true), b(false)))
+ }
+
+ def program2[E[_]](using alg: Algebra[E]): E[Int] = {
+ import alg._
+ sum(i(24), i(-3))
+ }
+}
+```
+
+But this implementation can very well be the description of a realistic service, e.g. a user-login service:
+
+```scala3
+object TaglessFinal_V2 {
+ trait UserLogin[E[_]] {
+ def checkLogin(mfa: Boolean): E[Boolean]
+ def countActiveSessions(server: Int): E[Int]
+ def mfa_v1(email: E[Boolean], sms: E[Boolean]): E[Boolean]
+ def mfa_v2(phone: E[Boolean], mobileApp: E[Boolean]): E[Boolean]
+ def totalSessionLogins(server1Logins: E[Int], server2Logins: E[Int]): E[Int]
+ }
+
+ case class UserLoginStatus[A](value: A)
+ given loginCapabilityImplementation: UserLogin[UserLoginStatus] with {
+ override def checkLogin(mfa: Boolean) = UserLoginStatus(mfa)
+ override def countActiveSessions(server: Int) = UserLoginStatus(server)
+ override def mfa_v1(email: UserLoginStatus[Boolean], sms: UserLoginStatus[Boolean]) = UserLoginStatus(email.value || sms.value)
+ override def mfa_v2(phone: UserLoginStatus[Boolean], mobileApp: UserLoginStatus[Boolean]) = UserLoginStatus(phone.value && mobileApp.value)
+ override def totalSessionLogins(server1Logins: UserLoginStatus[Int], server2Logins: UserLoginStatus[Int]) = UserLoginStatus(server1Logins.value + server2Logins.value)
+ }
+
+ def userLoginFlow[E[_]](using alg: UserLogin[E]): E[Boolean] = {
+ import alg._
+ mfa_v1(checkLogin(true), mfa_v2(checkLogin(true), checkLogin(false)))
+ }
+
+ def checkLastStatus[E[_]](using alg: UserLogin[E]): E[Int] = {
+ import alg._
+ totalSessionLogins(countActiveSessions(24), countActiveSessions(3))
+ }
+}
+```
+
+Yes, it's the exact same code with different function names — we see this all the time in service descriptions based on the Cats Effect/Typelevel stack!
+
+## 6. Conclusion
+
+In this article, we visited the tagless final approach, what it involves and what it means for us as Scala programmers. If you now know
+
+- where TF comes from
+- that TF is just an extension of "programming to interfaces"
+- that TF is not the same concept as `F[_]` + type classes, but only similar in representation
+
+then I've done my job and this article (and the video above) is a success.
diff --git a/_posts/2021-12-28-doobie.md b/_posts/2021-12-28-doobie.md
new file mode 100644
index 000000000000..cbe9bc9da8ee
--- /dev/null
+++ b/_posts/2021-12-28-doobie.md
@@ -0,0 +1,978 @@
+---
+title: "Learn Doobie for the Greater Good"
+date: 2021-12-28
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "We rely on the JDBC specification if we need to connect to SQL databases in the JVM ecosystem. However, JDBC is not a good fit if we use functional programming since the library performs a lot of side effects. A library called Doobie provides a higher-level API on top of JDBC, using an effectful style through the Cats and Cats Effect libraries."
+---
+
+_This article is brought to you by Riccardo Cardin, a proud student of the [Scala with Cats course](https://rockthejvm.com/p/cats) and guest contributor to the blog. Riccardo is a senior developer, a teacher and a passionate technical blogger. He's on his way to mastering functional programming and in he's constantly outdoing himself with his coverage of various libraries in his articles. We worked together over a month on this blog post._
+
+_We hope you enjoy it!_
+
+The vast majority of applications today connect with some form of a persistent layer, and, sooner or later, every developer faces the challenge of connecting to a database. We rely on the JDBC specification if we need to connect to SQL databases in the JVM ecosystem. However, JDBC is not a good fit if we use functional programming since the library performs a lot of side effects. Fortunately, a library called [Doobie](https://tpolecat.github.io/doobie/) provides a higher-level API on top of JDBC, using an effectful style through the Cats and Cats Effect libraries.
+
+So, without further ado, let's introduce the Doobie library.
+
+> This article uses advanced Scala features. We teach these in the [Advanced Scala](https://rockthejvm.com/p/advanced-scala) course.
+
+## 1. Set Up
+
+As usual, we'll start by importing the libraries we need in the SBT file. We will use Postgres as our database of reference:
+
+```scala
+val DoobieVersion = "1.0.0-RC1"
+val NewTypeVersion = "0.4.4"
+
+libraryDependencies ++= Seq(
+ "org.tpolecat" %% "doobie-core" % DoobieVersion,
+ "org.tpolecat" %% "doobie-postgres" % DoobieVersion,
+ "org.tpolecat" %% "doobie-hikari" % DoobieVersion,
+ "io.estatico" %% "newtype" % NewTypeVersion
+)
+```
+
+As we said, **Doobie is a library that lives in the Cats ecosystem**. However, the dependencies to Cats and Cats Effects are already contained in the Doobie library. Version 1.0.0-RC1 of Doobie uses the Cats Effect version 3.
+
+Since we chose to use Postgres as our database, we need to spin up a Postgres instance. We will use the [Postgres Docker image](https://hub.docker.com/r/postgres/postgres/) to do this. To simplify the process, we define the Postgres image in a _docker-compose.yml_ file:
+
+```yaml
+version: '3.1'
+
+services:
+ db:
+ image: postgres
+ restart: always
+ volumes:
+ - "./sql:/docker-entrypoint-initdb.d"
+ environment:
+ - "POSTGRES_USER=docker"
+ - "POSTGRES_PASSWORD=docker"
+ ports:
+ - "5432:5432"
+ adminer:
+ image: adminer
+ restart: always
+ ports:
+ - 8080:8080
+```
+
+The above configuration defines a Postgres instance listening on port 5432 and having a user, _admin_, with password _example_. We mount a directory `sql` containing the SQL scripts to be executed on startup, and that we will define in a moment.
+
+Moreover, we define an Adminer instance listening on port 8080. Adminer is a web interface to Postgres, which we will use to create a database, some tables, and populate them with some data.
+
+Next, we need a use case to train our skills about Doobie. We will use the same use case we introduced in the article [Unleashing the Power of HTTP Apis: The Http4s Library](https://blog.rockthejvm.com/http4s-tutorial/), which is implementing a small IMDB-like web service. The primary domain objects of the service are movies, actors, and directors. The goal is to use Doobie to interact with these tables through queries, insertion, and updates.
+
+Inside Postgres, we will model the domain objects as tables and define their relations as foreign keys. The tables will be named `movies`, `actors`, and `directors`:
+
+```sql
+-- Database
+CREATE DATABASE myimdb;
+\c myimdb;
+
+-- Directors
+CREATE TABLE directors (
+ id serial NOT NULL,
+ PRIMARY KEY (id),
+ name character varying NOT NULL,
+ last_name character varying NOT NULL
+);
+
+-- Movies
+CREATE TABLE movies (
+ id uuid NOT NULL,
+ title character varying NOT NULL,
+ year_of_production smallint NOT NULL,
+ director_id integer NOT NULL
+);
+
+ALTER TABLE movies
+ADD CONSTRAINT movies_id PRIMARY KEY (id);
+ALTER TABLE movies
+ADD FOREIGN KEY (director_id) REFERENCES directors (id);
+
+-- Actors
+CREATE TABLE actors (
+ id serial NOT NULL,
+ PRIMARY KEY (id),
+ name character varying NOT NULL
+);
+
+-- Link between movies and actors
+CREATE TABLE movies_actors (
+ movie_id uuid NOT NULL,
+ actor_id integer NOT NULL
+);
+
+ALTER TABLE movies_actors
+ADD CONSTRAINT movies_actors_id_movies_id_actors PRIMARY KEY (movie_id, actor_id);
+ALTER TABLE movies_actors
+ADD FOREIGN KEY (movie_id) REFERENCES movies (id);
+ALTER TABLE movies_actors
+ADD FOREIGN KEY (actor_id) REFERENCES actors (id);
+```
+
+The ER diagram associated with the above tables' definition is the following:
+
+
+
+Then, we need to populate the above tables with some data. As in the previous article, we renew our love for the movie "Zack Snyder's Justice League":
+
+```sql
+-- Actors
+INSERT INTO actors (name) VALUES ('Henry Cavill');
+INSERT INTO actors (name) VALUES ('Gal Godot');
+INSERT INTO actors (name) VALUES ('Ezra Miller');
+INSERT INTO actors (name) VALUES ('Ben Affleck');
+INSERT INTO actors (name) VALUES ('Ray Fisher');
+INSERT INTO actors (name) VALUES ('Jason Momoa');
+COMMIT;
+
+-- Directors
+INSERT INTO directors (name, last_name)
+VALUES ('Zack', 'Snyder');
+COMMIT;
+
+-- Movies
+INSERT INTO movies (id, title, year_of_production, director_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 'Zack Snyder''s Justice League', '2021', 1);
+COMMIT;
+
+-- Actor-Movie link
+INSERT INTO movies_actors (movie_id, actor_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 1);
+INSERT INTO movies_actors (movie_id, actor_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 2);
+INSERT INTO movies_actors (movie_id, actor_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 3);
+INSERT INTO movies_actors (movie_id, actor_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 4);
+INSERT INTO movies_actors (movie_id, actor_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 5);
+INSERT INTO movies_actors (movie_id, actor_id)
+VALUES ('5e5a39bb-a497-4432-93e8-7322f16ac0b2', 6);
+COMMIT;
+```
+
+In Scala, we will use the following classes to define the domain objects:
+
+```scala
+case class Actor(id: Int, name: String)
+
+case class Movie(id: String, title: String, year: Int, actors: List[String], director: String)
+
+// We will define the Director class later in the article. NO SPOILER!
+```
+
+Finally, all the examples contained in the article will use the following imports:
+
+```scala
+import cats.data.NonEmptyList
+import cats.effect._
+import cats.implicits._
+import doobie._
+import doobie.implicits._
+import io.estatico.newtype.macros.newtype
+import java.util.UUID
+// Very important to deal with arrays
+import doobie.postgres._
+import doobie.postgres.implicits._
+import doobie.util.transactor.Transactor._
+```
+
+So, with the above solid background, we can now enter the world of Doobie.
+
+## 2. Getting a Connection
+
+The first thing we need to work on within a database is retrieving a connection. In Doobie, handling the connection is done with a `doobie.util.transactor.Transactor`. There are many ways to create an instance of a `Transactor`. The easiest is to use the `Transactor.fromDriverManager` method, which will create a `Transactor` from a JDBC driver manager:
+
+```scala
+val xa: Transactor[IO] = Transactor.fromDriverManager[IO](
+ "org.postgresql.Driver",
+ "jdbc:postgresql:myimdb",
+ "docker", // username
+ "docker" // password
+)
+```
+
+This approach is the most straightforward, but it is not the most efficient. The reason is that the JDBC driver manager will try to load the driver for each connection, which can be pretty expensive. Moreover, the driver manager has no upper bound on the number of connections it will create. However, while experimenting and testing Doobie features, this approach works quite well.
+
+As we said in the introduction, Doobie is just a wrapper over the JDBC specification in Java, and it uses the Cats Effect library under the hood. Since **JDBC provides only a blocking interface** to interact with SQL databases, we should be careful to also use the blocking facilities available in Cats Effect. Fortunately, **Doobie takes care of using the `Blocking` context for us**:
+
+```scala
+// Doobie library's code
+// The ev variable is an instance of Async[IO]
+val acquire = ev.blocking{ Class.forName(driver); conn() }
+```
+
+In production code, as we said, we don't want to use an instance of `Transactor` coming directly from the JDBC driver manager. Instead, we will use a `Transactor` that is backed by a connection pool. Doobie integrates well with the [HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool library through the `doobie-hikari` module. Since a connection pool is a resource with its own lifecycle, we will use the Cats Effect `Resource` type to manage it:
+
+```scala
+val postgres: Resource[IO, HikariTransactor[IO]] = for {
+ ce <- ExecutionContexts.fixedThreadPool[IO](32)
+ xa <- HikariTransactor.newHikariTransactor[IO](
+ "org.postgresql.Driver",
+ "jdbc:postgresql:myimdb",
+ "postgres",
+ "example", // The password
+ ce
+ )
+} yield xa
+```
+
+We will directly use the `Transactor` coming from the JDBC driver manager in most article examples. In the last part, we will focus on using the `Resource` type to manage the connection pool.
+
+## 3. Querying the Database
+
+### 3.1. High-Level Queries
+Now that we learned how to connect to a database, we can start querying it. The most straightforward query we can do is to retrieve all actors names in the database, since the query doesn't request any input parameter, and extract only one column:
+
+```scala
+def findAllActorsNamesProgram: IO[List[String]] = {
+ val findAllActorsQuery: doobie.Query0[String] = sql"select name from actors".query[String]
+ val findAllActors: doobie.ConnectionIO[List[String]] = findAllActorsQuery.to[List]
+ findAllActors.transact(xa)
+}
+```
+
+As it's the first query we make, the code is really verbose. However, we can analyze every aspect of a query in this way.
+
+First, the `sql` [interpolator](https://blog.rockthejvm.com/how-to-create-your-own-string-interpolator/) allows us to create SQL statement fragments (more to come). Next, the method `query` lets us create a type that maps the single-row result of the query in a Scala type. The class is called `Query0[A]`. To accumulate results into a list, we use the `to[List]` method, which creates a `ConnectionIO[List[String]]`.
+
+The `ConnectionIO[A]` type is interesting since it introduces a typical pattern used in the Doobie library. In fact, **Doobie defines all its most essential types as instances of the [`Free` monad](https://typelevel.org/cats/datatypes/freemonad.html)**.
+
+Although the description and profound comprehension of the free monad is behind the scope of this article, we can say that a program with the type `ConnectionIO[A]` represents a computation that, given a `Connection`, will generate a value of type `IO[A]`.
+
+Every free monad is only a description of a program. It's not executable at all since it requires an interpreter. The interpreter, in this case, is the `Transactor` we created. Its role is to compile the program into a `Kleisli[IO, Connection, A]`. The course on [Cats](https://rockthejvm.com/p/cats) explains `Kleisli` in depth, but in short, the previous `Kleisli` is another representation of the function `Connection => IO[A]`.
+
+So, given an instance of `IO[Connection]` to the `Kleisli` through the `transact` method, we can execute the compiled program into the desired `IO[A]`, and then run it using the Cats Effect library:
+
+```scala
+object DoobieApp extends IOApp {
+
+ val xa: Transactor[IO] = Transactor.fromDriverManager[IO](
+ "org.postgresql.Driver",
+ "jdbc:postgresql:myimdb",
+ "docker",
+ "docker"
+ )
+
+ def findAllActorsNamesProgram: IO[List[String]] = {
+ val findAllActorsQuery: doobie.Query0[String] = sql"select name from actors".query[String]
+ val findAllActors: doobie.ConnectionIO[List[String]] = findAllActorsQuery.to[List]
+ findAllActors.transact(xa)
+ }
+
+ override def run(args: List[String]): IO[ExitCode] = {
+ findAllActorsNamesProgram()
+ .map(println)
+ .as(ExitCode.Success)
+ }
+}
+```
+
+Given the information we initially stored in the database, the above code produces the following output:
+
+```
+List(Henry Cavill, Gal Godot, Ezra Miller, Ben Affleck, Ray Fisher, Jason Momoa)
+```
+
+If we know for sure that the query will return exactly one row, we can use the `unique` method:
+
+```scala
+def findActorByIdProgram(id: Int): IO[Actor] = {
+ val findActorById: doobie.ConnectionIO[Actor] =
+ sql"select id, name from actors where id = $id".query[Actor].unique
+ findActorById.transact(xa)
+}
+```
+
+However, if the query doesn't return any row, we will get an exception. So, we can safely use the `option` method and let the program return an `Option[Actor]`:
+
+```scala
+def findActorByIdProgram(id: Int): IO[Option[Actor]] = {
+ val findActorById: doobie.ConnectionIO[Option[Actor]] =
+ sql"select id, name from actors where id = $id".query[Actor].option
+ findActorById.transact(xa)
+}
+```
+
+Although extracting actors in a `List[String]` seems legit at first sight, it's not safe in a real-world scenario. In fact, the number of extracted rows could be too much for the memory allocated to the application. For this reason, we should use a `Stream` instead of a `List`. **Doobie integrates smoothly with the functional streaming library [fs2](https://fs2.io)**. Again, describing how fs2 works is behind the scope of this article, and we just focus on how to use it with Doobie.
+
+For example, let's change the above example to use the streaming API:
+
+```scala
+val actorsNamesStream: fs2.Stream[doobie.ConnectionIO, String] =
+ sql"select name from actors".query[String].stream
+```
+
+The `stream` method on the type `Query0` returns a `Stream[ConnectionIO, A]` which means a stream containing instances of type `A`, wrapped in an effect of type `ConnectionIO` (for a brief explanation of the Effect pattern, please refer to [The Effect Pattern](https://blog.rockthejvm.com/zio-fibers/#2-the-effect-pattern)).
+
+Once we obtained an instance of a `Stream`, we can decide to return it to the caller as it is, or to compile it into a finite type, such as `List[String]` or `Vector[String]`:
+
+```scala
+val actorsNamesList: IO[List[String]] = actorsNamesStream.compile.toList.transact(xa)
+```
+
+Clearly, there are more than single-column queries. In fact, Doobie can handle multi-column queries as well. For example, let's query the ids and names of all the actors and return them as a tuple:
+
+```scala
+def findAllActorsIdsAndNamesProgram: IO[List[(Int, String)]] = {
+ val query: doobie.Query0[(Int, String)] = sql"select id, name from actors".query[(Int, String)]
+ val findAllActors: doobie.ConnectionIO[List[(Int, String)]] = query.to[List]
+ findAllActors.transact(xa)
+}
+```
+
+We can map the query result inside an instance of an `HList` or inside an `Option`. However, as we can imagine, the most helpful mapping of returned columns is directly into a class. Let's say we want to store the information of extracted actors directly into instances of the class `Actor` class:
+
+```scala
+def findAllActorsProgram: IO[List[Actor]] = {
+ val findAllActors: fs2.Stream[doobie.ConnectionIO, Actor] =
+ sql"select id, name from actors".query[Actor].stream
+ findAllActors.compile.toList.transact(xa)
+}
+```
+
+**Doobie can map the tuple of extracted columns directly into a `case class`**. For now, let's assume that the mapping between the extracted tuple and the properties of the case class must be one-to-one. In the last part of the article, we will introduce the type classes that allow the conversion of a tuple into a case class.
+
+One last thing about selecting information from a table is to add parameters to the query. Fortunately, the `sql` interpolator works smoothly with parameters, using the exact mechanism used by Scala native `String` interpolation:
+
+```scala
+def findActorsByNameInitialLetterProgram(initialLetter: String): IO[List[Actor]] = {
+ val findActors: fs2.Stream[doobie.ConnectionIO, Actor] =
+ sql"select id, name from actors where LEFT(name, 1) = $initialLetter".query[Actor].stream
+ findActors.compile.toList.transact(xa)
+}
+```
+
+The above program extracts all `actors` from the table whose names start with the given initial letter. As we can see, passing a parameter to a query is as simple as passing it to an interpolated string.
+
+### 3.2. The `HC` Module
+
+The interpolator `sql` is a very handy syntax for writing SQL queries. However, it's not the only way to write queries. In fact, it's just an API, implemented in terms of the functions available in the `doobie.hi.connection` module, aliased as `HC`.
+
+So, let's take the first program we developed above, the `findAllActorsNamesProgram`, and desugar it using the `HC` module:
+
+```scala
+def findActorByNameUsingHCProgram(actorName: String): IO[Option[Actor]] = {
+ val query = "select id, name from actors where name = ?"
+ HC.stream[Actor](
+ query,
+ HPS.set(actorName), // Parameters start from index 1 by default
+ 512
+ ).compile
+ .toList
+ .map(_.headOption)
+ .transact(xa)
+}
+```
+
+First, the query becomes a plain `String` containing `?` wildcards. The `sql` interpolator is just syntactic sugar for the `HC.stream[A]` method. Leaving the comprehension of the first parameter to the reader, the second parameter has type `PreparedStatementIO[B]`. As for the `ConnectionIO[A]` type, a `PreparedStatementIO` is an instance of the free monad pattern. In this case, it describes how to inject parameters into the query. So, the interpreter of the monad, `HC`, builds a `Kleisli[IO, PreparedStatement, A]`, which is precisely a function returning an `IO[A]` given a `PreparedStatement`.
+
+The third parameter is the maximum number of rows to be fetched at a time. In fact, Doobie reads rows in chunks.
+
+If the query has more than one parameter, we have many choices on the syntax to use:
+
+```scala
+// Set parameters as (Int, String)
+HPS.set((1, "Henry Cavill"))
+
+// Set parameters individually
+HPS.set(1, 1) *> HPS.set(2, "Henry Cavill")
+
+// ...and many others!
+```
+
+### 3.2. Fragments
+
+Until now, we used the `sql` interpolator to build our queries. It turns out that the `sql` interpolator is an alias of the more general `fr` interpolator, whose name stands for `Fragment. **A fragment is a piece of an SQL statement that we can combine with any other fragment to build a proper SQL instruction**.
+
+Imagine building the query at runtime, extracting the list of actors whose names start with a given initial letter. Using fragments, we can do it as follows:
+
+```scala
+def findActorsByInitialLetterUsingFragmentsProgram(initialLetter: String): IO[List[Actor]] = {
+ val select: Fragment = fr"select id, name"
+ val from: Fragment = fr"from actors"
+ val where: Fragment = fr"where LEFT(name, 1) = $initialLetter"
+
+ val statement = select ++ from ++ where
+
+ statement.query[Actor].stream.compile.toList.transact(xa)
+}
+```
+
+In the example above, we build the three parts of the SQL statements, and then we combine them to produce the final query using the `++` operator. It's easy to understand why `Fragment` is also a `Monoid` since it's possible to use the `++` operator to define the `combine` function of monoids (more on the article [Semigroups and Monoids in Scala](https://blog.rockthejvm.com/semigroups-and-monoids-in-scala/)):
+
+```scala
+// Doobie library's code
+object fragment {
+ implicit val FragmentMonoid: Monoid[Fragment] =
+ new Monoid[Fragment] {
+ val empty = Fragment.empty
+
+ def combine(a: Fragment, b: Fragment) = a ++ b
+ }
+}
+```
+
+So, if we want to use the combination operator `|+|`, made available by the monoid instance, we can rewrite the previous example as follows:
+
+```scala
+def findActorsByInitialLetterUsingFragmentsAndMonoidsProgram(initialLetter: String): IO[List[Actor]] = {
+ import cats.syntax.monoid._
+
+ val select: Fragment = fr"select id, name"
+ val from: Fragment = fr"from actors"
+ val where: Fragment = fr"where LEFT(name, 1) = $initialLetter"
+
+ val statement = select |+| from |+| where
+
+ statement.query[Actor].stream.compile.toList.transact(xa)
+}
+```
+
+As we say, fragments are helpful to build queries dynamically. One widespread use case is to create a query that uses the `IN` operator since JDBC doesn't give any built-in support for this kind of operator. Fortunately, Doobie does it, providing a reliable method in the `Fragments` object:
+
+```scala
+def findActorsByNamesProgram(actorNames: NonEmptyList[String]): IO[List[Actor]] = {
+ val sqlStatement: Fragment =
+ fr"select id, name from actors where " ++ Fragments.in(fr"name", actorNames) // name IN (...)
+
+ sqlStatement.query[Actor].stream.compile.toList.transact(xa)
+}
+```
+
+Indeed, the `Fragments` object contains a lot of valuable functions to implement many recurring SQL queries patterns:
+
+```scala
+// Doobie library's code
+val Fragments = doobie.util.fragments
+object fragments {
+ /** Returns `(f1) AND (f2) AND ... (fn)`. */
+ def and(fs: Fragment*): Fragment = ???
+
+ /** Returns `(f1) OR (f2) OR ... (fn)`. */
+ def or(fs: Fragment*): Fragment = ???
+
+ /** Returns `WHERE (f1) AND (f2) AND ... (fn)` or the empty fragment if `fs` is empty. */
+ def whereAnd(fs: Fragment*): Fragment = ???
+
+ /** Returns `WHERE (f1) OR (f2) OR ... (fn)` or the empty fragment if `fs` is empty. */
+ def whereOr(fs: Fragment*): Fragment = ???
+
+ // And many more...
+}
+```
+
+## 4. The YOLO Mode
+
+While experimenting with the library, passing the `Transactor` instance around could seem a little overwhelming. Moreover, the syntax `transact(xa)` is a bit cumbersome. Adding that it's widespread to print out the program's results to the console, the library will help us do that. Please, welcome the YOLO mode!
+
+First, we need a stable reference to the `Transactor` instance. Then we can import the `yolo` module:
+
+```scala
+val y = xa.yolo
+import y._
+```
+
+Then, imagine we want to find all actors in the database and print them out. Using the `yolo` syntax, we can write the following program:
+
+```scala
+object YoloApp extends App {
+
+ import cats.effect.unsafe.implicits.global
+
+ val xa: Transactor[IO] = Transactor.fromDriverManager[IO](
+ "org.postgresql.Driver",
+ "jdbc:postgresql:myimdb",
+ "docker",
+ "docker"
+ )
+
+ val y = xa.yolo
+ import y._
+
+ val query = sql"select name from actors".query[String].to[List]
+ query.quick.unsafeRunSync()
+}
+```
+
+As we can see, the program doesn't need an `IOApp` to execute. Then, the `quick` method is syntactic sugar for calling the `transact` method, which is a bit more verbose and then sinking the stream to standard output.
+
+Remember, You Only Load Once!
+
+## 5. Not Only Queries: Changing the Database
+
+The other side of the moon of the database world is mutating the tables and the data they contain. Doobie offers support not only for queries but also for insertions, updates, deletions, and changes on tables' structure.
+
+It should not come as a surprise that **inserting follows the same pattern of selecting rows from tables**. For example, let's save a new actor inside the `actors` table:
+
+```scala
+def saveActorProgram(name: String): IO[Int] = {
+ val saveActor: doobie.ConnectionIO[Int] =
+ sql"insert into actors (name) values ($name)".update.run
+ saveActor.transact(xa)
+}
+```
+
+As we can see, we continue to use the `sql` interpolator and its capabilities of dealing with input parameters. However, the `update` method returns an instance of the `Update0` class. It corresponds to the `Query0` class in the case of DMLs.
+
+We need to call one available method to get a `ConnectionIO` from an `Update0`. The easiest way to do this is to call the `run` method, which returns the number of updated rows inside the `ConnectionIO`.
+
+Moreover, it's possible to get back the autogenerated id of the inserted row, using the `withUniqueGeneratedKeys` method and specifying the column name:
+
+```scala
+def saveActorAndGetIdProgram(name: String): IO[Int] = {
+ val saveActor: doobie.ConnectionIO[Int] =
+ sql"insert into actors (name) values ($name)"
+ .update.withUniqueGeneratedKeys[Int]("id")
+ saveActor.transact(xa)
+}
+```
+
+Be careful: Only some databases natively implement this feature, such as H2 or Postgres. Moreover, the method `withUniqueGeneratedKeys` will fail if the number of modified rows is more than one.
+
+Now that we know how to insert and retrieve information from a table, we can create a program that both inserts and retrieves an actor in sequence:
+
+```scala
+def saveAndGetActorProgram(name: String): IO[Actor] = {
+ val retrievedActor = for {
+ id <- sql"insert into actors (name) values ($name)".update.withUniqueGeneratedKeys[Int]("id")
+ actor <- sql"select * from actors where id = $id".query[Actor].unique
+ } yield actor
+ retrievedActor.transact(xa)
+}
+```
+
+Here, we use the fact that the type `ConnectionIO[A]` is a _monad_, which means we can chain operations on it through a sequence of calls to `flatMap` and `map` methods.
+
+Doobie allows us to insert more than one row at a time. However, to do so, we have to desugar the `sql` interpolator and the subsequent call to the `run` method. So, let's first remove the syntactic sugar:
+
+```scala
+val name = "John Travolta"
+
+// This statement...
+sql"insert into actors (name) values ($name)".update.run
+
+// ...is equivalent to this one:
+val stmt = "insert into actors (name) values (?)"
+Update[String](stmt).run(name)
+```
+
+Now, we can use the desugared version of the `sql` interpolator to insert multiple rows. As an example, we can insert a list of actors into the database:
+
+```scala
+def saveActorsProgram(actors: NonEmptyList[String]): IO[Int] = {
+ val insertStmt: String = "insert into actors (name) values (?)"
+ val numberOfRows: doobie.ConnectionIO[Int] = Update[String](insertStmt).updateMany(actors.toList)
+ numberOfRows.transact(xa)
+}
+```
+
+As we can see, Doobie gives us a method, `updateMany`, to execute a batch insertion. It takes a list of parameters and returns the number of rows inserted. Some databases, such as Postgres, can produce a list of column values of the inserted rows. In this case, we can use the method `updateManyWithGeneratedKeys`, taking as input a list of column names:
+
+```scala
+def saveActorsAndReturnThemProgram(actors: NonEmptyList[String]): IO[List[Actor]] = {
+ val insertStmt: String = "insert into actors (name) values (?)"
+ val actorsIds = Update[String](insertStmt).updateManyWithGeneratedKeys[Actor]("id", "name")(actors.toList)
+ actorsIds.compile.toList.transact(xa)
+}
+```
+
+Updating information in the database is the same affair as inserting. There are no substantial differences between the two. Imagine we want to fix the year of production of the movie "Zack Snyder's Justice League". Here is how we can do it:
+
+```scala
+def updateJLYearOfProductionProgram(): IO[Int] = {
+ val year = 2021
+ val id = "5e5a39bb-a497-4432-93e8-7322f16ac0b2"
+ sql"update movies set year_of_production = $year where id = $id".update.run.transact(xa)
+}
+```
+
+Finally, the deletion follows the same pattern. Use the keyword `delete` instead of `insert` or `update` inside the `sql` interpolator.
+
+## 6. Doobie's Type Classes
+
+So far, we have seen many examples of usages of the `sql` interpolator, which magically can convert Scala types into JDBC types when reading input parameters, and vice versa when concerning mapping values extracted from the database.
+
+As we can imagine, there is no magic whatsoever. As skilled Scala developers, we should have known that **there are some type classes behind it whenever someone talks about magic**.
+
+In fact, Doobie basically uses four type classes for the conversion between Scala and JDBC types: `Get[A]`, `Put[A]`, `Read[A]` and `Write[A]`.
+
+The `Get[A]` describes creating the Scala type `A` from a non-nullable database value. We can also apply the same type class to obtain type `Option[A]`. So, Doobie uses an instance of `Get[A]` to map the results of a query into Scala.
+
+The `Put[A]` type class describes creating a non-nullable database value from the Scala type `A`. As for the `Get[A]`, the `Put[A]` type class maps also the `Option[A]` type.
+
+Doobie defines the instances of the above type classes for the following types (directly from the [Doobie documentation](https://tpolecat.github.io/doobie/docs/12-Custom-Mappings.html)):
+
+* JVM numeric types `Byte`, `Short`, `Int`, `Long`, `Float`, and `Double`;
+* `BigDecimal` (both Java and Scala versions);
+* `Boolean`, `String`, and `Array[Byte]`;
+* `Date`, `Time`, and `Timestamp` from the `java.sql` package;
+* `Date` from the `java.util` package;
+* `Instant`, `LocalDate`, `LocalTime`, `LocalDateTime`, `OffsetTime`, `OffsetDateTime` and `ZonedDateTime` from the `java.time` package; and
+* single-element case classes wrapping one of the above types.
+
+Deriving the `Get` and `Put` type classes for types that don't fit into one of the above categories is relatively easy. To create a concrete example, we introduce in our project the [estatico/scala-newtype](https://github.com/estatico/scala-newtype), which allows creating a new type that is a subtype of the original type but with a different name. The description of newtypes is far beyond the scope of this article, but you can find a good introduction on [Value Classes in Scala](https://blog.rockthejvm.com/value-classes/).
+
+First, let's create a newtype wrapper around an actor name:
+
+```scala
+@newtype case class ActorName(value: String)
+```
+
+Now, we can try to use the newtype to map the result of a query:
+
+```scala
+def findAllActorNamesProgram(): IO[List[ActorName]] = {
+ sql"select name from actors".query[ActorName].to[List].transact(xa)
+}
+```
+
+As we can expect, when we try to compile the above code, we get an error since the compiler cannot find any suitable type classes instances for the `ActorName` type:
+
+```
+[error] Cannot find or construct a Read instance for type:
+[error]
+[error] DoobieApp.ActorName
+[error]
+[error] This can happen for a few reasons, but the most common case is that a data
+[error] member somewhere within this type doesn't have a Get instance in scope. Here are
+[error] some debugging hints:
+[error]
+[error] - For Option types, ensure that a Read instance is in scope for the non-Option
+[error] version.
+[error] - For types you expect to map to a single column ensure that a Get instance is
+[error] in scope.
+[error] - For case classes, HLists, and shapeless records ensure that each element
+[error] has a Read instance in scope.
+[error] - Lather, rinse, repeat, recursively until you find the problematic bit.
+[error]
+[error] You can check that an instance exists for Read in the REPL or in your code:
+[error]
+[error] scala> Read[Foo]
+[error]
+[error] and similarly with Get:
+[error]
+[error] scala> Get[Foo]
+[error]
+[error] And find the missing instance and construct it as needed. Refer to Chapter 12
+[error] of the book of doobie for more information.
+[error] sql"select name from actors".query[ActorName].to[List].transact(xa)
+```
+
+Fortunately, Doobie gives us all the tools to quickly create such type classes. The first method we can use is to **derive the `Get[ActorName]` and `Put[ActorName]` type classes from the same defined for the `String` type**:
+
+```scala
+object ActorName {
+ implicit val actorNameGet: Get[ActorName] = Get[String].map(ActorName(_))
+ implicit val actorNamePut: Put[ActorName] = Put[String].contramap(actorName => actorName.value)
+}
+```
+
+As we may imagine, the `map` method is defined in the `Functor[Get]` type class, whereas the `contramap` method is defined in the `Contravariant[Put]` type class:
+
+```scala
+// Doobie library's code
+trait GetInstances extends GetPlatform {
+ implicit val FunctorGet: Functor[Get] =
+ new Functor[Get] {
+ def map[A, B](fa: Get[A])(f: A => B): Get[B] =
+ fa.map(f)
+ }
+}
+
+trait PutInstances extends PutPlatform {
+ implicit val ContravariantPut: Contravariant[Put] =
+ new Contravariant[Put] {
+ def contramap[A, B](fa: Put[A])(f: B => A): Put[B] =
+ fa.contramap(f)
+ }
+}
+```
+
+Moreover, in the case of newtypes, we can simplify the definition of the `Get` and `Put` type classes, using the `deriving` method available in the newtype library:
+
+```scala
+@newtype case class ActorName(value: String)
+object ActorName {
+ implicit val actorNameGet: Get[ActorName] = deriving
+ implicit val actorNamePut: Put[ActorName] = deriving
+}
+```
+
+Since it's widespread to derive both type classes for a type, Doobie also defines the `Meta[A]` type class, which allows us to create with a single statement both the `Get` and the `Put` type classes for a type:
+
+```scala
+implicit val actorNameMeta: Meta[ActorName] = Meta[String].imap(ActorName(_))(_.value)
+```
+
+The first parameter of the `imap` method is the function that defines the `Get` instance, whereas the second parameter is the function that defines the `Put` instance. The `imap` function comes from the `Invariant[Meta]` type class defined in the Cats library.
+
+Again, since the newtype library defines the `deriving` method, we can simplify the definition of the `Meta` type class:
+
+```scala
+implicit val actorNameMeta: Meta[ActorName] = deriving
+```
+
+**Doobie uses the `Get` and `Put` type classes only to manage single-column schema types**. In general, we need to map more than one column directly into a Scala class or into a tuple. For this reason, Doobie defines two more type classes, `Read[A]` and `Write[A]`, which can handle heterogeneous collections of columns.
+
+The `Read[A]` allows us to map a vector of schema types inside a Scala type. Vice versa, the `Write[A]` will enable us to interact with more complex types instead of plain ints, strings etc.
+
+Logically, **the `Read` and `Write` type classes are defined as a composition of `Get` and `Put` on the attributes of the referenced type**. In detail, the type can be a record (tuple), or a product type (a case class), or even an `HList`. In addition, Doobie adds the mapping of `Option[A]`.
+
+In any other case, we must define a custom mapping, as we have previously done. Starting from the definition of `Read` and `Write` for a well-known type, we use the `map` and `contramap` functions to derive the needed type classes. Speaking about our movies' database, imagine we want to read from the `directors` table. We can define the Scala type `Director` representing a single row of the table:
+
+```scala
+object domain {
+ @newtype case class DirectorId(id: Int)
+ @newtype case class DirectorName(name: String)
+ @newtype case class DirectorLastName(lastName: String)
+
+ case class Director(id: DirectorId, name: DirectorName, lastName: DirectorLastName)
+}
+```
+
+Since we are well-grounded Scala developers, we defined a newtype wrapping every native type. Now, we want to get all the directors stored in the table:
+
+```scala
+def findAllDirectorsProgram(): IO[List[Director]] = {
+ val findAllDirectors: fs2.Stream[doobie.ConnectionIO, Director] =
+ sql"select id, name, last_name from directors".query[Director].stream
+ findAllDirectors.compile.toList.transact(xa)
+}
+```
+
+However, as we may expect, running the above program generates an error since the compiler cannot find any type class instance for `Read[Director]`. So, let's define the missing type class instances:
+
+```scala
+object Director {
+ implicit val directorRead: Read[Director] =
+ Read[(Int, String, String)].map { case (id, name, lastname) =>
+ Director(DirectorId(id), DirectorName(name), DirectorLastName(lastname))
+ }
+ implicit val directorWrite: Write[Director] =
+ Write[(Int, String, String)].contramap { director =>
+ (director.id.id, director.name.name, director.lastName.lastName)
+ }
+}
+```
+
+_Et voilà_, now the program runs without any error.
+
+## 7. Handling Joins
+
+Until now, we presented some very straightforward examples of queries. However, we can also handle joins. The good news is that Doobie takes join between tables in a very natural way.
+
+Let's say we want to find a movie by its name. We want to retrieve also the director's information and the list of actors that played in the film. Using the ER model we presented at the beginning of the article, we have to join three tables: the `movies` table, the `directors` table, and the `actors` table. Here is how we can implement it in Doobie:
+
+```scala
+def findMovieByNameProgram(movieName: String): IO[Option[Movie]] = {
+ val query = sql"""
+ |SELECT m.id,
+ | m.title,
+ | m.year_of_production,
+ | array_agg(a.name) as actors,
+ | d.name
+ |FROM movies m
+ |JOIN movies_actors ma ON m.id = ma.movie_id
+ |JOIN actors a ON ma.actor_id = a.id
+ |JOIN directors d ON m.director_id = d.id
+ |WHERE m.title = $movieName
+ |GROUP BY (m.id,
+ | m.title,
+ | m.year_of_production,
+ | d.name,
+ | d.last_name)
+ |""".stripMargin
+ .query[Movie]
+ .option
+ query.transact(xa)
+}
+```
+
+Since the `actor` table extracts many rows for every movie, we used a `GROUP BY` operation and the `array_agg` Postgres function, which creates an array from the actors' names.
+
+However, the array type is not SQL standard. So, to let Doobie map the array type to a Scala `List`, we need to import the `doobie.postgres._` and `doobie.postgres.implicits._` packages, belonging to the `doobie-postgres` library.
+
+As we said, the array type is not standard, and the database we're using might not support arrays. In this case, the only solution left is to **perform the join manually, which means splitting the original query into three different queries and joining the data programmatically**:
+
+```scala
+def findMovieByNameWithoutSqlJoinProgram(movieName: String): IO[Option[Movie]] = {
+
+ def findMovieByTitle() =
+ sql"""
+ | select id, title, year_of_production, director_id
+ | from movies
+ | where title = $movieName"""
+ .query[(UUID, String, Int, Int)].option
+
+ def findDirectorById(directorId: Int) =
+ sql"select name, last_name from directors where id = $directorId"
+ .query[(String, String)].to[List]
+
+ def findActorsByMovieId(movieId: UUID) =
+ sql"""
+ | select a.name
+ | from actors a
+ | join movies_actors ma on a.id = ma.actor_id
+ | where ma.movie_id = $movieId
+ |""".stripMargin
+ .query[String]
+ .to[List]
+
+ val query = for {
+ maybeMovie <- findMovieByTitle()
+ directors <- maybeMovie match {
+ case Some((_, _, _, directorId)) => findDirectorById(directorId)
+ case None => List.empty[(String, String)].pure[ConnectionIO]
+ }
+ actors <- maybeMovie match {
+ case Some((movieId, _, _, _)) => findActorsByMovieId(movieId)
+ case None => List.empty[String].pure[ConnectionIO]
+ }
+ } yield {
+ maybeMovie.map { case (id, title, year, _) =>
+ val directorName = directors.head._1
+ val directorLastName = directors.head._2
+ Movie(id.toString, title, year, actors, s"$directorName $directorLastName")
+ }
+ }
+ query.transact(xa)
+}
+```
+
+In the above code, we extracted information from the `movies` table, the `directors` table, and the `actors` table in sequence, and then we mapped the data to a `Movie` object. The `ConnectionIO` type is a monad. We can compose the queries in a sequence using the _for-comprehension_ construct. Even though it's not the main focus of this code, as we said, the three queries are executed in a single database transaction.
+
+## 8. Putting Pieces Together: A Tagless Final Approach
+
+Now that we know all the pieces of a program that connects to a database using Doobie, we can create a more complex example. For this purpose, we will use the _tagless final_ approach. Again, the details of the tagless final approach are far beyond the scope of this tutorial. In summary, it's **a technique that allows us to manage dependencies between our components and abstract away the details of the concrete effect implementation**.
+
+> More on tagless final in the [dedicated article](/tagless-final).
+
+In a tagless final approach, we first define an _algebra_ as a `trait`, storing all the functions we implement for a type. If we take the `Director` type, we can define the following algebra:
+
+```scala
+trait Directors[F[_]] {
+ def findById(id: Int): F[Option[Director]]
+ def findAll: F[List[Director]]
+ def create(name: String, lastName: String): F[Int]
+}
+```
+
+As we can see, we abstract away from the concrete effect implementation, replacing it with a type constructor `F[_]`.
+
+Then, we need an _interpreter_ of the algebra, that is a concrete implementation of the functions defined in the algebra:
+
+```scala
+object Directors {
+ def make[F[_]: MonadCancelThrow](xa: Transactor[F]): Directors[F] = {
+ new Directors[F] {
+ import DirectorSQL._
+
+ def findById(id: Int): F[Option[Director]] =
+ sql"SELECT id, name, last_name FROM directors WHERE id = $id".query[Director].option.transact(xa)
+
+ def findAll: F[List[Director]] =
+ sql"SELECT id, name, last_name FROM directors".query[Director].to[List].transact(xa)
+
+ def create(name: String, lastName: String): F[Int] =
+ sql"INSERT INTO directors (name, last_name) VALUES ($name, $lastName)".update.withUniqueGeneratedKeys[Int]("id").transact(xa)
+ }
+ }
+}
+```
+
+Following the approach suggested by Gabriel Volpe in his excellent book, [Practical FP in Scala: A hands-on approach](https://leanpub.com/pfp-scala), we use a _smart constructor_ to create an instance of the interpreter that we maintain private to the rest of the world. The implementation of every single function should not surprise us at this point. It's just the same Doobie code we've seen so far.
+
+Since the `Transactor` is something our interpreter depends on, and it's not related to any feature concerning the effect `F[_]`, we can safely pass it as an explicit parameter of the smart constructor.
+
+The `Transactor` type has a hard constraint on the effect `F[_]`. In fact, the effect should be an instance of at least a `MonadCancelThrow`, which is a monad that can effectively handle errors and is cancellable. The last property allows the monad to safely cancel executions and release / close resources.
+
+We use context-bounds directly on `F` to define any constraint on the effect.
+
+Finally, we can create a `Directors` instance using the smart constructor.
+
+As we should remember, since we've defined the `Director` type through the use of many newtypes, we need to create a custom mapping using the `Read` and `Write` type classes. We can put the type classes definition inside a dedicated object:
+
+```scala
+private object DirectorSQL {
+ implicit val directorRead: Read[Director] =
+ Read[(Int, String, String)].map { case (id, name, lastname) =>
+ Director(DirectorId(id), DirectorName(name), DirectorLastName(lastname))
+ }
+
+ implicit val directorWrite: Write[Director] =
+ Write[(Int, String, String)].contramap { director =>
+ (director.id.id, director.name.name, director.lastName.lastName)
+ }
+}
+```
+
+If we want to create a concrete instance of our interpreter, we need to create an instance of the `Transactor` resource. As we saw at the beginning of this article, we can use the Hikary extension for Doobie:
+
+```scala
+val postgres: Resource[IO, HikariTransactor[IO]] = for {
+ ce <- ExecutionContexts.fixedThreadPool[IO](32)
+ xa <- HikariTransactor.newHikariTransactor[IO](
+ "org.postgresql.Driver",
+ "jdbc:postgresql:myimdb",
+ "postgres",
+ "example",
+ ce
+ )
+} yield xa
+```
+
+Then, we can define the last part of the tagless final approach: a _program_ using the interpreter. For example, let's define a program that inserts the director of Jurassic Park into the database and then retrieve it. Note how we're creating the `Directors[F]` instance, by calling `use` on the postgres resource, thereby reusing the transactor on all queries.
+
+```scala
+val program: IO[Unit] = postgres.use { xa =>
+ val directors = Directors.make[IO](xa)
+ for {
+ id <- directors.create("Steven", "Spielberg")
+ spielberg <- directors.findById(id)
+ _ <- IO.println(s"The director of Jurassic Park is: $spielberg")
+ } yield ()
+}
+```
+
+We can then put all the parts into a single `IOApp` and run the program:
+
+```scala
+object TaglessApp extends IOApp {
+ override def run(args: List[String]): IO[ExitCode] = {
+ val postgres: Resource[IO, HikariTransactor[IO]] = for {
+ ce <- ExecutionContexts.fixedThreadPool[IO](32)
+ xa <- HikariTransactor.newHikariTransactor[IO](
+ "org.postgresql.Driver",
+ "jdbc:postgresql:myimdb",
+ "docker",
+ "docker",
+ ce
+ )
+ } yield xa
+
+ val program: IO[Unit] = postgres.use { xa =>
+ val directors = Directors.make[IO](xa)
+ for {
+ id <- directors.create("Steven", "Spielberg")
+ spielberg <- directors.findById(id)
+ _ <- IO.println(s"The director of Jurassic Park is: $spielberg")
+ } yield ()
+ }
+
+ program.as(ExitCode.Success)
+ }
+}
+```
+
+And that's it!
+
+## 9. Conclusions
+
+Finally, we sum up what we've learned so far. We introduced Doobie, a JDBC functional wrapper library built upon the Cats Effect library. After defining some domain models to work with, we learned how to create a `Transactor` object to execute instructions in the database. Then, we saw how to implement queries, both with and without input parameters, and map their results back to our domain models. So, we saw how to insert and update rows in a table, and then which are the available implementation when we need a join. Since Doobie uses some type classes to map Scala type from and to schema types, we introduced them. Finally, we describe how to use Doobie in a tagless final context with all the pieces in the right places.
+
+Clearly, the article is not exhaustive, but it's an excellent start understanding how to use Doobie. We left out some advanced features, like testing, error handling, type-checking, and logging. If you want to deepen your knowledge concerning the Doobie library, you can take a look at the [official documentation](https://tpolecat.github.io/doobie/).
diff --git a/_posts/2021-12-31-the-2021-retrospective.md b/_posts/2021-12-31-the-2021-retrospective.md
new file mode 100644
index 000000000000..f768f0995865
--- /dev/null
+++ b/_posts/2021-12-31-the-2021-retrospective.md
@@ -0,0 +1,74 @@
+---
+title: "A 2021 Retrospective"
+date: 2021-12-31
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [meta]
+excerpt: "Wrapping up 2021 with an overview of what's happened this year at Rock the JVM."
+---
+
+2021 has been an interesting year to say the least. In this last article for the year, I'd like to share the progress, experiments, mishaps and successes at Rock the JVM over the year, as well as hopes for what Rock the JVM can become.
+
+## 1. The Blog
+
+Last year, I promised the blog would add a new article every week, give or take. Out of 52 weeks of the year, the blog added 36 articles, which means roughly an article every week and a half. Although not terrible, this rate could be improved. However, the scale, length and depth of the articles improved dramatically, and I'm very happy with the rate + depth combination of the blog this year, compared to last year.
+
+By far the most significant improvement of the blog was in welcoming guest posts. [Riccardo Cardin](https://github.com/rcardin) quickly became a regular contributor, with an in-depth article every month or so. [Giannis Polyzos](https://github.com/polyzos) also contributed with an introduction to Pulsar, which is growing like crazy and quickly becoming a sought-after piece of technology. I suspect we'll see more in-depth coverage from these folks, I'm super proud of them and of what we've managed to publish.
+
+What I'm hoping for: if I were to choose between a faster pace and more in-depth stuff, I would choose more in-depth stuff. I intend to cover most of the Scala ecosystem over the Rock the JVM website (i.e. courses) and the blog (i.e. articles, tutorials), and there's plenty more to write about.
+
+If you're interested in contributing for the Rock the JVM blog, [contact me](mailto:daniel@rockthejvm.com).
+
+## 2. YouTube
+
+Wherever possible, I also recorded video versions of the blog posts over at Rock the JVM's [YouTube channel](https://youtube.com/rockthejvm). The channel grew more than 3x in the past year to 9535 subscribers as of this writing, which for a Scala-focused code-with-me programming channel, I believe it's quite remarkable.
+
+The videos got overwhelmingly positive feedback, with more than 9 out of 10 videos getting 100% likes and thousands of views. Thank you for spending your most precious resource (time) listening to me. I'll keep shooting more videos and keeping the parity with the written form (the blog). I'm also talking to a few bigger partners who are interested in my content, so you'll probably see some more breadth of coverage as well, possibly extending beyond Scala.
+
+Unlike the blog, I kept the sole responsibility for publishing videos. People come to Rock the JVM for my style of teaching, and I intend to continue being the voice of Rock the JVM for the YouTube channel (and of course, the courses).
+
+What I'm hoping for:
+
+- that the partnerships I'm forming will help a lot of people (outside my email/subscriber list) in a meaningful way
+- that more people will find Rock the JVM through my content (is 2x subscriber growth still possible?)
+
+## 3. Socials
+
+By far my most active socials are [Twitter](https://twitter.com/rockthejvm) and [LinkedIn](https://linkedin.com/company/rockthejvm). This is where I see the most activity, the most engagement and the most feedback.
+
+The Rock the JVM Twitter follower list has grown more than 4x to more than 2800 subscribers. I try to tweet every day, either with punchline lessons, Scala jokes, news or just fooling around. Some tweets fell flat, others were received quite well. [This tweetstorm](https://twitter.com/rockthejvm/status/1379695298365300736) was by far my best Twitter creation.
+
+I should post more on LinkedIn, though. The engagement is crazy, even on simple posts.
+
+## 4. Courses!
+
+This year, I published courses at roughly the same pace as last year. The highlights are
+
+- [Scala & FP Interview Practice](https://rockthejvm.com/p/scala-functional-programming-interview-practice), which now doubled in size
+- [Scala 3 Essentials](https://rockthejvm.com/p/scala) and
+- [Scala 3 Advanced](https://rockthejvm.com/p/advanced-scala), both completely re-recorded and remastered for Scala 3
+- [Cats Effect](https://rockthejvm.com/p/cats-effect)
+
+All in all, the site now contains more than 200 hours of content and 20000+ lines of code written from scratch on camera. I haven't counted the recorded time I scrapped or the code that will not see the light of day.
+
+I was quite bummed with the pace this year, as I hoped I would double last year's performance. People have been asking for Akka Typed for more than a year, for example, and I always had to postpone it. Same with a lot of other requests.
+
+Hopes for next year: fill the gaps (e.g. Akka Typed), expand breadth (e.g. courses on data engineering) and cover more of the Scala library ecosystem (ZIO anyone?).
+
+## 5. Corporate Training
+
+This was rock solid. I was booked for training sessions from _every single one_ of my major clients that you can see on the main page. I'm very happy and proud to have taught hard topics to some of the best companies in the world. New clients include Intuit and Orange, and my big ones, including Apple and Adobe, came back for new (and challenging) sessions this year. Everything remote, of course, because of the pandemic.
+
+I also pushed myself. For instance, I taught an Advanced Spark session in Python, which was way outside my comfort zone; the feedback forms scored a "perfect" 5.0/5, although I could personally see a lot of ways to improve.
+
+The downside of company training sessions is that they require a lot of prep. Sadly, I had to spend more than 1/3 of the year just for training sessions, which kept me from releasing new content for everyone else. I'll tweak the balance more in favor of courses and new content in the new year.
+
+That said, if you'd like to have me hold a course for your team/your company, let me know at [daniel@rockthejvm.com](mailto:daniel@rockthejvm.com).
+
+## 6. Thank You
+
+With all the uncertainty in the air, Rock the JVM has grown a lot. And this is due to you, dear readers, listeners, and students of Rock the JVM. Thank you for supporting me and all my work on this tiny site — I'm determined to make 2022 the best year yet for Rock the JVM!
+
+For now, enjoy some respite with your loved ones and let's get back with some fresh energy in the new year.
+Enjoy!
+Daniel
diff --git a/_posts/2022-01-20-pulsar-spark.md b/_posts/2022-01-20-pulsar-spark.md
new file mode 100644
index 000000000000..ba084bf22d08
--- /dev/null
+++ b/_posts/2022-01-20-pulsar-spark.md
@@ -0,0 +1,417 @@
+---
+title: "Streaming Analytics with Apache Pulsar and Spark Structured Streaming"
+date: 2022-01-29
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [pulsar, spark, spark structured streaming]
+excerpt: "We'll discuss the role Apache Pulsar plays in the event streaming and computing landscape, the use cases you can implement with Pulsar and when more advanced computing engines should be integrated to implement more sophisticated stream processing use cases."
+---
+
+_This article is a collaboration between me (Daniel) and [Giannis Polyzos](https://github.com/polyzos), one of the earliest students of Rock the JVM back in 2017. Giannis is now a senior engineer and a contributor to Apache Pulsar, a promising new toolkit for distributed messaging and streaming. In this piece we combine two of our favorite pieces of tech: Apache Pulsar and Apache Spark._
+
+Stream processing is an important requirement in modern data infrastructures. Companies now aim to leverage the power of streaming and real-time analytics in order to provide results faster to their users in order to enhance the user experience and drive business value.
+
+Examples are limitless: think of an online service that wants to provide user recommendations based on the interactions the user makes while on the webpage; or an IoT company that wants to monitor the sensor readings and in order to react on potential malfunctions; or computer vision systems that need to analyze realtime video feeds or fraud detection in banking systems; and the list
+goes on and on.
+
+Typically, streaming data pipelines require a streaming storage layer like Apache Pulsar or Apache Kafka, and then in order to perform more sophisticated stream processing tasks we need a stream compute engine like Apache Flink or Spark Structured Streaming.
+
+The requirement of unified batch processing and streaming in cloud deployments makes Apache Pulsar an excellent candidate to support the needs of these compute engines. Apache Pulsar is designed for cloud-native infrastructure as well as a unified batch and stream data strategy in mind.
+
+## 1. The Role of Apache Pulsar in Streaming Data Pipelines
+
+> Apache Pulsar excels at storing event streams and performing lightweight stream computing tasks. It's a great fit for long term storage of data and can also be used to store results to some downstream applications.
+
+A unified batch and stream processing engine requires the support from a unified batch and stream storage layer to achieve its full feature set.
+
+In other words, the streaming system needs to not only move data in the pipelines, but to store data in streams for long periods of time and serves as the single source of truth for the processing engine.
+
+Apache Pulsar’s tiered storage architecture enables it to store data streams indefinitely.
+
+Pulsar is a great fit for long term storage of data and can be used to store results to some downstream applications.
+
+Apache Pulsar is a distributed append-only log that can easily scale out. Topics or topic partitions can be spread across multiple brokers, supporting high throughput parallel reads of topics across multiple machines and partitions.
+
+Pulsar topics also employ a segment-based architecture, which means that each of these individual topics also has a sequence of segments with segments periodically closing and becoming immutable.
+
+These segments are stored in the underlying mechanism of Apache BookKeeper, but can be offloaded to long-term, inexpensive storage, such as Amazon S3, leveraging Pulsar’s tiered storage.
+
+This flexibility also makes Pulsar’s underlying storage architecture ideal for batch access. Historical data can be stored and accessed in a batch mechanism, with the appropriate stream processing layer on top. Pulsar offers both low-latency and high bandwidth streaming APIs, with architecture capable of supporting modern data infrastructures seeking to unify Batch and Stream Processing. Pulsar Functions is (are?) a lightweight serverless stream processing framework - but note the term **lightweight** here.
+
+Apache Pulsar isn’t an advanced stream computing engine but gives you the flexibility to implement many common stream processing use cases; let’s zoom a little more on that.
+
+With Pulsar Functions you can implement common use cases on streaming data, like:
+
+- Simple data aggregations, e.g. counters
+- Routing patterns:
+ - Dynamic or Content-based routing
+ - Splitter pattern
+- Message Transformations:
+ - Data masking for privacy concerns
+ - Data enrichment from external systems
+ - Validation of data payloads
+ - Content filtering
+- Realtime Machine Learning model deployment and scoring
+
+'Lightweight' also means that Apache Pulsar is an excellent candidate for _edge analytics_ — with limited resources, devices can benefit from such a framework and analytics can truly be taken right to the data collection for simple analytical tasks like univariate/multivariate time series analysis.
+
+But for use cases that require more sophisticated computations such as
+
+- Grouping aggregates
+- Joining different data streams
+- Using advanced time windowing to perform computations
+- Support for advanced watermarks for handling late arriving events
+- Handling large amounts of state
+
+we should leverage sophisticated stream compute engines like Spark Structured Streaming and Apache Flink. Pulsar aims to provide an excellent integration with them, so you can leverage their advanced capabilities to a full extent.
+
+## 2. Example Use Case: Real-Time User Engagement
+
+Let’s take a closer look now at an example use case to better understand the role of all these different components in a streaming data pipeline.
+
+Imagine a popular e-commerce website; the users click on different products, add products to their carts and (hopefully) they purchase the items. From a data-driven business perspective, we can imagine a few things we can do by leveraging stream processing:
+
+1. If the user is a subscribed customer that collects loyalty points: Upon a new purchase we might want to calculate in real time the total loyalty points they have collected and offer a big discount for their next purchase.
+2. If the user added multiple items to the cart, clicked the cart, but never actually completed checkout, we might want to send a followup email (maybe with some small discount coupon code if they is not a registered user) to remind them to complete the order.
+
+Here is a sample visual representation of this data pipeline:
+
+
+
+We have events being generated from the website, including user information and product information, as well the user click events. Notice that the topics storing product and user information are compacted, which means that we specify a key for the topic and that key is each user’s (or product’s) unique identifier.
+
+We are only interested in the latest values of those records and a compacted topic provides just that: having the latest update for each record based on the key.
+
+> Take a look at the Spark Job that joins multiple data streams. Now imagine for a moment that instead of having the user and product information inside Pulsar topics, we'd had them stored in an external database, such as MySQL.
+
+In this scenario, a Pulsar Function that reads the records and performs a lookup in the external system to grab the necessary user and product information could be a better solution.
+
+## 3. Using the Apache Pulsar/Spark Connector
+
+Let's take our example use case and zoom-in the abandoned cart implementation by combining Apache Pulsar and Spark Structured Streaming.
+
+In order to integrate Apache Pulsar and Apache Spark, we will use the [pulsar-spark connector](https://github.com/streamnative/pulsar-spark) - developed and maintained by StreamNative.
+
+The first thing we need to do is setup the dependencies within our project - so let's add the following to our `build.sbt` file:
+
+```scala
+lazy val sparkVersion = "3.2.0"
+lazy val circeVersion = "0.14.1"
+lazy val pulsar4sVersion = "2.8.1"
+lazy val pulsarSparkVersion = "3.1.1.3"
+
+val ingestionDependencies = Seq(
+ "com.clever-cloud.pulsar4s" % "pulsar4s-core_2.12" % pulsar4sVersion,
+ "com.clever-cloud.pulsar4s" % "pulsar4s-circe_2.12" % pulsar4sVersion
+)
+
+val analysisDependencies = Seq(
+ "io.circe" %% "circe-core" % circeVersion,
+ "io.circe" %% "circe-generic" % circeVersion,
+ "io.circe" %% "circe-parser" % circeVersion,
+ "org.apache.spark" %% "spark-sql" % sparkVersion,
+ "io.streamnative.connectors" % "pulsar-spark-connector_2.12" % pulsarSparkVersion
+)
+
+libraryDependencies ++= ingestionDependencies ++ analysisDependencies
+```
+
+Imagine we have the following click events inside Pulsar (synthetically generated for this example):
+
+```shell
+----- got message -----
+key:[536638900], properties:[], content:{"eventTime":1572559255,"eventType":"view","productId":1307519,"categoryId":2053013558920217191,"categoryCode":"computers.notebook","brand":"acer","price":"1209.55","userId":536638900,"userSession":"e1e8125d-da26-49ee-a6fa-78b3ff8dc341"}
+----- got message -----
+key:[532364121], properties:[], content:{"eventTime":1572559256,"eventType":"view","productId":12708937,"categoryId":2053013553559896355,"categoryCode":"","brand":"michelin","price":"72.72","userId":532364121,"userSession":"0a899268-31eb-46de-898d-09b2da950b24"}
+----- got message -----
+key:[513998949], properties:[], content:{"eventTime":1572559256,"eventType":"view","productId":50600085,"categoryId":2134905044833666047,"categoryCode":"auto.accessories.compressor","brand":"laston","price":"113.93","userId":513998949,"userSession":"a7b196d9-afe5-4dc8-9648-d578fef55abf"}
+----- got message -----
+key:[544501248], properties:[], content:{"eventTime":1572559256,"eventType":"view","productId":1003316,"categoryId":2053013555631882655,"categoryCode":"electronics.smartphone","brand":"apple","price":"928.38","userId":544501248,"userSession":"e330d051-37ad-4dc3-b1ee-ff16a28b7998"}
+----- got message -----
+key:[515240495], properties:[], content:{"eventTime":1572559256,"eventType":"view","productId":30000218,"categoryId":2127425436764865054,"categoryCode":"construction.tools.welding","brand":"magnetta","price":"254.78","userId":515240495,"userSession":"0253151d-5c84-4809-ba02-38ac405494e1"}
+----- got message -----
+key:[566280567], properties:[], content:{"eventTime":1572559258,"eventType":"view","productId":1004322,"categoryId":2053013555631882655,"categoryCode":"electronics.smartphone","brand":"huawei","price":"334.37","userId":566280567,"userSession":"8cd74350-34e7-423b-ab02-53108a89354b"}
+----- got message -----
+key:[559033632], properties:[], content:{"eventTime":1572559258,"eventType":"view","productId":22700084,"categoryId":2053013556168753601,"categoryCode":"","brand":"force","price":"244.28","userId":559033632,"userSession":"fe9544f7-0c09-4c85-a2f7-1d978a2710be"}
+----- got message -----
+key:[531148230], properties:[], content:{"eventTime":1572559259,"eventType":"view","productId":1004767,"categoryId":2053013555631882655,"categoryCode":"electronics.smartphone","brand":"samsung","price":"242.63","userId":531148230,"userSession":"57a91bce-a1fb-4609-bb26-d53430dc6618"}
+```
+
+This can be modeled in Scala as events of the following form:
+
+```scala
+case class Event(userid: String,
+ eventTime: Long,
+ eventType: String,
+ productId: String,
+ categoryId: String,
+ categoryCode: String,
+ brand: String,
+ price: Double,
+ userSession: String)
+
+object Event {
+ def empty(): Event = Event("", 0L, "" , "", "", "", "", 0.0, "")
+}
+```
+
+For our simple example implementation, we will consider a cart as _abandoned_, if inside a user session we have `cart` events but no `purchase` events. We are going to push events to Pulsar, then read and analyze them through Spark Structured Streaming.
+
+Let's take them in turn.
+
+### 3.1. Producer: Pulsar
+
+We'll simulate these events as we described in the [Pulsar intro](/event-streaming-with-pulsar-and-scala/) article. First, we'll set up a Docker container that will run Pulsar:
+
+```shell
+docker run -it \
+ -p 6650:6650 \
+ -p 8080:8080 \
+ --name pulsar \
+ apachepulsar/pulsar:2.8.0 \
+ bin/pulsar standalone
+```
+
+And in another terminal we'll create a topic `events` to accept our values:
+
+```shell
+docker exec -it pulsar bash
+bin/pulsar-admin topics create events
+```
+
+Now, in our Scala application, we need to set up a Pulsar producer. Assuming we can [parse our data easily](#33-parsing-events) (in this case from a file with randomized, realistic data), we'll need to set up the necessary Pulsar scaffolding:
+
+```scala
+import com.sksamuel.pulsar4s.{DefaultProducerMessage, EventTime, MessageId, ProducerConfig, PulsarClient, Topic}
+import com.sksamuel.pulsar4s.circe._
+import io.circe.generic.auto._
+import io.circe.{Decoder, Encoder, Json, Printer}
+import org.apache.pulsar.client.api.Schema
+import org.apache.pulsar.common.schema.{SchemaInfo, SchemaType}
+
+lazy val topicName = "events"
+lazy val eventsFilePath = "src/main/resources/events.csv"
+lazy val serviceUrl = "pulsar://localhost:6650"
+lazy val adminUrl = "http://localhost:8080"
+lazy val producerName = "event-producer"
+lazy val threadPoolSize = 4
+
+// events parsed elsewhere
+val events: List[Event] = loadEvents(eventsFilePath, withHeader = true)
+
+val pulsarClient = PulsarClient(serviceUrl)
+val topic = Topic(topicName)
+
+val eventProducer = pulsarClient.producer[Event](
+ ProducerConfig(
+ topic,
+ producerName = Some(producerName),
+ enableBatching = Some(true),
+ blockIfQueueFull = Some(true)
+ )
+)
+```
+
+Then we need to process all events in the list by sending them asynchronously to Pulsar. The following is a quick sketch of how it would look like:
+
+```scala
+val messageIdFutures: Seq[Future[MessageId]] = events.map { event =>
+ Thread.sleep(10)
+
+ // marshal an event for Pulsar
+ val message: DefaultProducerMessage[Event] = DefaultProducerMessage[Event](
+ Some(event.userid),
+ event,
+ eventTime = Some(EventTime(event.eventTime)))
+
+ //send it
+ val messageIdFuture = eventProducer.sendAsync(message)
+
+ // being quite verbose to track messages
+ messageIdFuture.onComplete {
+ case Success(id) => println(s"Ack received for message with id '$id'.")
+ case Failure(exception) => println(s"Failed to sent message - Reason: $exception")
+ }
+
+ // keep the Future for analysis at the source, e.g. check message IDs
+ messageIdFuture
+}
+```
+
+We can then use the resulting Futures to track message IDs and whether they've been sent successfully or not, etc. For this demo, we're just going to close the application once it's done.
+
+```scala
+Future.sequence(messageIdFutures) // turn the Seq[Future[...]] into Future[Seq[...]]
+ .andThen {
+ case Success(_) => println("Producer finished sending event records.")
+ case Failure(e) => println(s"A failure occurred. Check the logs and the stack trace: $e")
+ }
+ .andThen {
+ case _ => eventProducer.close()
+ }
+```
+
+### 3.2. Consumer: Spark Structured Streaming
+
+Spark Structured Streaming — which we teach [here at Rock the JVM](https://rockthejvm.com/p/spark-streaming) — is a stream computing engine provides more advanced features that are helpful to our use case:
+
+- support for Session Windows - we can create session windows based on the eventTime and the userSession id.
+- support for groupBy aggregations - we want to aggregate the eventTypes on the user session windows.
+- watermarking support - allows for late event handling.
+- filtering on complex data types - filter rows based on a list filtering predicate.
+
+With our dependencies in place, let's first connect to Pulsar and start reading events from our `events` topic.
+
+```scala
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.streaming.OutputMode
+
+val clickEventsDF = spark
+ .readStream
+ .format("pulsar")
+ .option("service.url", "pulsar://localhost:6650")
+ .option("admin.url", "http://localhost:8080")
+ .option("topic", "events")
+ // the following two configs are optional for this demo but important if you're doing authenticated data access
+ .option("pulsar.client.authPluginClassName","org.apache.pulsar.client.impl.auth.AuthenticationToken")
+ .option("pulsar.client.authParams","token:you-token<>")
+ .load()
+```
+
+We can easily connect to Pulsar with Spark with the connection URLs, input topic name, and preferred security method if security is enabled. You can check the full list of the available configuration options at the [Pulsar connector's GitHub.](https://github.com/streamnative/pulsar-spark).
+
+With the connection in place, we'll transform the data to a custom data type useful for analysis:
+
+```scala
+case class AnalysisEvent(userSession: String, userId: String, eventTime: Timestamp, eventType: String)
+
+import spark.implicits._
+
+val parsedEvents = clickEventsDF
+ .selectExpr("CAST(value AS STRING)").as[String]
+ .map { line =>
+ val event = io.circe.parser.decode[Event](line).getOrElse(Event.empty())
+ AnalysisEvent(event.userSession, event.userid, new Timestamp(event.eventTime), event.eventType)
+ }.as[AnalysisEvent]
+```
+
+And with this Dataset in place, we can now sketch an example implementation:
+
+- discarding events older than 30 minutes
+- using session windows of 80 minutes, [now automatic in Spark 3.2](https://databricks.com/blog/2021/10/12/native-support-of-session-window-in-spark-structured-streaming.html)
+- aggregating event types
+- looking for shopping cart events
+
+```scala
+val checkoutEvents = parsedEvents
+ .withWatermark("eventTime", "30 minutes")
+ .groupBy(col("userSession"), col("userId"), session_window(col("eventTime"), "80 minutes"))
+ .agg(collect_list("eventType").as("eventTypes"))
+ .filter(array_contains(col("eventTypes"),"cart"))
+ .select(
+ col("session_window.start").as("sessionWindowStart"),
+ col("session_window.end").as("sessionWindowEnd"),
+ col("userId"),
+ col("userSession"),
+ col("eventTypes")
+ )
+```
+
+The output should look something like this:
+
+```shell
++-------------------+-------------------+---------+------------------------------------+------------------------------------------------------------------------------------------------------------------------------+
+|sessionWindowStart |sessionWindowEnd |userId |userSession |eventTypes |
++-------------------+-------------------+---------+------------------------------------+------------------------------------------------------------------------------------------------------------------------------+
+|2019-11-01 02:21:03|2019-11-01 02:56:28|564133858|ef95cac1-6580-44f0-a10b-c7e836982154|[view, view, cart, cart, purchase, view, view, view, cart, view] |
+|2019-11-01 00:59:55|2019-11-01 01:34:27|566286947|b86c0313-d0cd-4222-888f-2ed03a1d697f|[view, view, view, cart, purchase, view] |
+|2019-11-01 02:22:39|2019-11-01 02:54:45|528271890|b425dd2d-dae4-4915-8e8f-b1d4d2717471|[view, cart, view, view] |
+|2019-11-01 01:49:20|2019-11-01 02:21:31|553414803|5006939e-fbe7-4e1c-809c-ffa3b16eb20c|[view, view, view, cart, view] |
+|2019-11-01 01:54:59|2019-11-01 02:29:24|556325945|ad789442-8756-41d7-b05a-11b90124032d|[view, view, view, cart, purchase, view, cart, cart, cart, cart, cart, cart, cart, cart, cart, cart, cart] |
+|2019-11-01 00:37:42|2019-11-01 01:14:30|549630515|8847ab0c-1f0b-42fb-9ff4-f44b9f523b4b|[view, view, view, view, view, cart, view, view, view, view, cart, view, view, view] |
+|2019-11-01 00:13:32|2019-11-01 00:44:49|563558500|e0729b6c-eafe-4b0f-9d66-6ee777d08488|[view, cart, view, view, view, view, view] |
+|2019-11-01 02:15:30|2019-11-01 02:49:24|512411198|64a9195f-e4ee-448f-9241-9b4d23467f5d|[view, cart, view, view, cart, view] |
+|2019-11-01 01:44:05|2019-11-01 02:15:30|515742799|ca33e1b9-ecf5-4b50-ba76-007248bad43d|[view, cart, cart, cart, cart, view] |
+|2019-11-01 00:41:39|2019-11-01 01:12:36|557332447|70c0ccdf-9452-49cc-bfa5-568096d64680|[view, cart, view] |
+|2019-11-01 01:40:47|2019-11-01 02:34:50|520761071|6dad05da-a718-4f05-92bc-1ed9796404cd|[view, view, view, view, view, view, view, view, view, view, view, cart, purchase] |
+|2019-11-01 00:37:04|2019-11-01 01:08:52|562200921|ca5a71f1-33c8-4fcd-b793-5fcea6f260c0|[view, cart, purchase, view] |
+|2019-11-01 00:10:07|2019-11-01 00:46:10|516426931|ef4867dd-b922-4d92-abec-9a75acb2b769|[view, view, view, cart, purchase, view, cart, cart, view, view] |
+|2019-11-01 02:07:33|2019-11-01 02:45:30|554459781|c43b43dd-dc54-4d1c-bfd8-3bcbdfb94870|[view, cart, purchase, view, view] |
+|2019-11-01 02:11:34|2019-11-01 02:52:13|539100846|365b1088-8a4f-47e4-a6e9-e66d56b1b998|[view, cart, cart, cart, view, cart, view, view, view, view, view] |
+|2019-11-01 01:34:14|2019-11-01 02:08:51|519456951|f2136bd5-a50d-4e05-8a90-27b288065459|[view, cart, cart, purchase] |
+|2019-11-01 01:47:49|2019-11-01 02:26:31|515561413|98a12974-b589-48fe-b6f5-a55cd844cdd8|[view, view, view, view, cart, view] |
+|2019-11-01 01:29:06|2019-11-01 02:00:39|518913698|afa1ad69-55bb-4ef8-9d02-6648596ca5ec|[view, cart, purchase, view] |
+|2019-11-01 01:53:53|2019-11-01 02:36:26|556601011|45658f52-9e11-45fa-a8d6-9414d349aa4d|[view, view, view, view, view, view, cart, cart, view, view, view, view, view, view, view, view, view, view, view, view, view]|
+|2019-11-01 01:01:28|2019-11-01 01:34:04|538178630|9cf447dc-7aa8-47f2-8c3b-44318cf5608a|[view, view, view, cart, cart, cart, purchase, view] |
++-------------------+-------------------+---------+------------------------------------+------------------------------------------------------------------------------------------------------------------------------+
+```
+
+Some notes on session windows:
+
+- A Session window sizes dynamically based on the window length, depending on the inputs.
+- A Session window starts with an input, and expands itself if the following input is received within the gap duration.
+- For a static gap duration, a session window closes when there’s no input received within gap duration after receiving the latest input.
+
+By using session windows and grouping on the user sessions, we can easily see if a user has a cart event type, but no purchase event. This makes it easy to filter the events we consider as **abandoned cart** and forward them to a downstream topic for further processing. An example action could be to send out a reminder notification (via smartphone app if available, or via email) to the users after some period of time, e.g. an hour.
+
+### 3.3. Parsing Events
+
+This bit should not be the front and center of the article as the focus is on Pulsar and Spark, so for your convenience, this is the simple code we used to parse the events in [this csv file](https://raw.githubusercontent.com/polyzos/pulsar-spark-structured-streaming/main/data/events.csv).
+
+```scala
+import java.sql.Timestamp
+import java.util.concurrent.{ExecutorService, Executors}
+import scala.concurrent.{ExecutionContext, Future}
+import scala.io.BufferedSource
+import scala.util.{Failure, Success, Try}
+
+def loadEvents(path: String, withHeader: Boolean = false): List[Event] = {
+ val r = Try(scala.io.Source.fromFile(path))
+ .map(source => sourceHandler(source, withHeader))
+ .fold(_ => List.empty, identity)
+
+ val result = Try(scala.io.Source.fromFile(path)) match {
+ case Success(source) =>
+ Some(sourceHandler(source, withHeader))
+ case Failure(exception) =>
+ println(s"Failed to read file '$path' - Reason: $exception")
+ None
+ }
+
+ result.getOrElse(List.empty)
+}
+
+private def sourceHandler(source: BufferedSource, withHeader: Boolean): List[Event] = {
+ val events: List[Event] = source.getLines()
+ .map(toEvent)
+ .toList
+
+ if (withHeader) events.drop(1) else events
+}
+
+private def toEvent(line: String): Event = {
+ Try {
+ val tokens = line.split(",")
+ val eventTime = Timestamp.valueOf(tokens(0).replace(" UTC", ""))
+
+ Event(
+ tokens(7),
+ eventTime.getTime,
+ tokens(1),
+ tokens(2),
+ tokens(3),
+ tokens(4),
+ tokens(5),
+ tokens(6).toDouble,
+ tokens(8)
+ )
+ }.getOrElse(Event.empty())
+}
+```
+
+## 4. Conclusion
+
+In this article we discussed the role of Apache Pulsar as a backbone of a modern data infrastructure, the streaming use cases Pulsar can support, and how you can use it along with Spark Structured Streaming to implement some more advanced stream processing use cases by leveraging the Pulsar Spark Connector.
+
+We also reviewed a real world use case, demonstrated a sample streaming data pipeline, and examined the role of Apache Pulsar and Spark Structured Streaming within the pipeline.
diff --git a/_posts/2022-02-10-fs2.md b/_posts/2022-02-10-fs2.md
new file mode 100644
index 000000000000..0cc4a532ddb2
--- /dev/null
+++ b/_posts/2022-02-10-fs2.md
@@ -0,0 +1,675 @@
+---
+title: "FS2 Tutorial: More than Functional Streaming in Scala"
+date: 2022-02-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [typelevel, fs2]
+excerpt: "The ultimate tutorial to purely functional streams in Scala with FS2."
+---
+
+_Another long-form by [Riccardo Cardin](https://github.com/rcardin) - having started with the [Scala with Cats course](https://rockthejvm.com/p/cats) a while back, he's now steadily mastering all the critical pieces of the Typelevel ecosystem._
+
+Nowadays, modern applications are often built on top of streaming data: Reading from a huge file and processing information or handling continuous data from the network as WebSockets or from a message broker. Streams are the way to go in such situations, and Scala provides its own streams implementation.
+
+However, streams in the standard library are not as powerful as they could be and don't offer concurrency, throttling, or backpressure features.
+
+Fortunately, some libraries offer more robust implementation of streams. One of these is the fs2 library, built on top of the Cats and Cats Effect libraries. Moreover, **fs2 provides an entirely functional approach to stream processing**. So, without further ado, let's dive into the details of fs2.
+
+## 1. Set Up
+
+The fs2 library is available both for Scala 2 and for Scala 3. The following code will set up SBT to use the library for Scala 3.
+
+```scala
+val Fs2Version = "3.2.4"
+libraryDependencies += "co.fs2" %% "fs2-core" % Fs2Version
+```
+
+As we said, **the fs2 streaming library is built on top of the Cats and Cats Effect libraries**. However, we don't need to specify them as direct dependencies in the sbt file since the two libraries are already contained in fs2.
+
+The `fs2-core` library provides the core functionalities of the library. Many other plugins add more features: Reading from and writing to file, from the network, and so on. Moreover, there are a lot of projects using fs2 under the hood, such as _http4s_, _doobie_, _skunk_, to name a few. Please, refer to the [fs2 documentation](https://fs2.io/#/ecosystem) for more information.
+
+To work well with fs2, you'll need to be comfortable with Scala, of course. Some basics of Cats Effect would be wonderful. Rock the JVM has this in-depth [Cats Effect course](https://rockthejvm.com/p/cats-effect) if you're interested in mastering it, but it's not required.
+
+It's usual for us at Rock the JVM to build the examples around a concrete scenario. We can continue to refer to the _myimdb_ project that we used both in the articles on [_http4s_](https://blog.rockthejvm.com/http4s-tutorial/) and on [_doobie_](https://blog.rockthejvm.com/doobie/).
+
+So, we define the Scala class that represents an actor inside a hypothetical movie database:
+
+```scala
+object Model {
+ case class Actor(id: Int, firstName: String, lastName: String)
+}
+```
+
+As we adore movies based on comics, we define some actors that are famous for playing the role of a hero:
+
+```scala
+object Data {
+ // Justice League
+ val henryCavil: Actor = Actor(0, "Henry", "Cavill")
+ val galGodot: Actor = Actor(1, "Gal", "Godot")
+ val ezraMiller: Actor = Actor(2, "Ezra", "Miller")
+ val benFisher: Actor = Actor(3, "Ben", "Fisher")
+ val rayHardy: Actor = Actor(4, "Ray", "Hardy")
+ val jasonMomoa: Actor = Actor(5, "Jason", "Momoa")
+
+ // Avengers
+ val scarlettJohansson: Actor = Actor(6, "Scarlett", "Johansson")
+ val robertDowneyJr: Actor = Actor(7, "Robert", "Downey Jr.")
+ val chrisEvans: Actor = Actor(8, "Chris", "Evans")
+ val markRuffalo: Actor = Actor(9, "Mark", "Ruffalo")
+ val chrisHemsworth: Actor = Actor(10, "Chris", "Hemsworth")
+ val jeremyRenner: Actor = Actor(11, "Jeremy", "Renner")
+ val tomHolland: Actor = Actor(13, "Tom", "Holland")
+ val tobeyMaguire: Actor = Actor(14, "Tobey", "Maguire")
+ val andrewGarfield: Actor = Actor(15, "Andrew", "Garfield")
+}
+```
+
+Finally, all the examples we are going to build will use the following imports:
+
+```scala
+import cats.effect.std.Queue
+import cats.effect.{ExitCode, IO, IOApp}
+import cats.syntax.all.*
+import fs2.{Chunk, INothing, Pipe, Pull, Pure, Stream}
+```
+
+## 2. Building a Stream
+
+As the official page of the fs2 stream library reports, its main features are:
+
+- Functional
+- Effectful
+- Concurrent
+- I/O (networking, files) computations in constant memory
+- Stateful transformations
+- Resource safety and effect evaluation
+
+Despite all the above features, **the primary type defined by the library is only one, `Stream[F, O]`. This type represents a stream that can pull values of type `O` using an effect of type `F`**. The last part of this sentence will be evident in a moment.
+
+The easiest way to create a `Stream` is to use its constructor directly. Say that we want to create a `Stream` containing the actors in the Justice League. We can do it like this:
+
+```scala
+val jlActors: Stream[Pure, Actor] = Stream(
+ henryCavil,
+ galGodot,
+ ezraMiller,
+ benFisher,
+ rayHardy,
+ jasonMomoa
+)
+```
+
+Since we don't use any effect to compute (or pull) the elements of the stream, we can use the `Pure` effect. In other words, **using the `Pure` effect means that pulling the elements from the stream cannot fail**.
+
+Similarly, we can create pure streams using smart constructors instead of the `Stream` constructor. Among the others, we have `emit` and `emits`, which create a pure stream with only one element or with a sequence of elements, respectively:
+
+```scala
+val tomHollandStream: Stream[Pure, Actor] = Stream.emit(tomHolland)
+val spiderMen: Stream[Pure, Actor] = Stream.emits(List(
+ tomHolland,
+ tobeyMaguire,
+ andrewGarfield
+))
+```
+
+As no effect is used at all, it's possible to convert _pure streams_ into Scala `List` of `Vector` using the convenient methods provided by the library:
+
+```scala
+val jlActorList: List[Actor] = jlActors.toList
+val jlActorVector: Vector[Actor] = jlActors.toVector
+```
+
+It's also possible to create also an infinite stream. The fs2 library provides some convenient methods to do this:
+
+```scala
+val infiniteJLActors: Stream[Pure, Actor] = jlActors.repeat
+val repeatedJLActorsList: List[Actor] = infiniteJlActors.take(12).toList
+```
+
+The `repeat` method does what its name suggests; it repeats the stream infinitely. Since we cannot put an infinite stream into a list, we take the stream's first _n_ elements and convert them into a list as we've done before.
+
+However, the `Pure` effect is not sufficient to pull new elements from a stream most of the time. In detail, **the operation can fail, or it must interact with some external resource or with some code performing _side effects_**. In this case, we need to use some effect library, such as Cats-effect, and its effect type, called `IO[A]`.
+
+The above are the "functional" and "effectful" parts. As we will see in a moment, **all the streams' definitions are referentially transparent** and remain pure since no side effects are performed.
+
+Starting from the stream we already defined, we can create a new effectful stream mapping the `Pure` effect in an `IO` effect using the `covary[F]` method:
+
+```scala
+val liftedJlActors: Stream[IO, Actor] = jlActors.covary[IO]
+```
+
+The method is called `covary` because of the covariance of the `Stream` type in the `F` type parameter:
+
+```scala
+// fs2 library code
+final class Stream[+F[_], +O]
+```
+
+Since the `Pure` effect is defined as an alias of the Scala bottom type `Nothing`, the `covary` method takes advantage of this fact and changes the effect type to `IO`:
+
+```scala
+// Covariance in F means that
+// Pure <: IO => Stream[Pure, O] <: Stream[IO, O]
+```
+
+It's not mandatory to use the `IO` effect. **We can use any other effect library** that supports type classes defined in the `cats-effect` library. As an example, let's use the `covary` method using a generic effect type:
+
+```scala
+def jlActorStream[F[_]: MonadThrow]: Stream[F, Actor] = jlActors.covary[F]
+```
+
+In this case, we chose to use the `MonadThrow` type class from the `cats-effect` library, which allows us to concatenate effects using the `flatMap` method and handle exceptions natively.
+
+In most cases, we want to create a stream directly evaluating some statements that may produce side effects. So, for example, let's try to persist an actor through a stream:
+
+```scala
+val savingTomHolland: Stream[IO, Unit] = Stream.eval {
+ IO {
+ println(s"Saving actor $tomHolland")
+ Thread.sleep(1000)
+ println("Finished")
+ }
+}
+```
+
+The fs2 library gives us the method `eval` that takes an `IO` effect and returns a `Stream` that will evaluate the `IO` effect when pulled. A question arises: How do we pull the values from an effectful stream? We cannot convert such a stream into a Scala collection using the `toList` function, and if we try, the compiler soundly yells at us:
+
+```shell
+[error] 95 | savingTomHolland.toList
+[error] | ^^^^^^^^^^^^^^^^^^^^^^^
+[error] |value toList is not a member of fs2.Stream[cats.effect.IO, Unit], but could be made available as an extension method.
+```
+
+In fs2 jargon, we need to _compile_ the stream into a single instance of the effect:
+
+```scala
+val compiledStream: IO[Unit] = savingTomHolland.compile.drain
+```
+
+In this case, we also applied the `drain` method, which discards any effect output. However, once compiled, we return to have many choices. For example, we can transform the compiled stream into an effect containing a `List`:
+
+```scala
+val jlActorsEffectfulList: IO[List[Actor]] = liftedJlActors.compile.toList
+```
+
+In the end, **compiling a stream always produce an effect of the type declared in the `Stream` first type parameter**. If we are using the `IO` effect, we can call the `unsafeRunSync` to run the effect:
+
+```scala
+import cats.effect.unsafe.implicits.global
+savingTomHolland.compile.drain.unsafeRunSync()
+```
+
+Otherwise, we can run our application as an `IOApp` (preferred way):
+
+```scala
+object Fs2Tutorial extends IOApp {
+ override def run(args: List[String]): IO[ExitCode] = {
+ savingTomHolland.compile.drain.as(ExitCode.Success)
+ }
+}
+```
+
+### 2.1. Chunks
+
+Inside, every stream is made of _chunks_. A `Chunk[O]` is a finite sequence of stream elements of type `O` stored inside a structure optimized for indexed based lookup of elements. We can create a stream directly through the `Stream.chunk` method, which accepts a sequence of `Chunk`:
+
+```scala
+val avengersActors: Stream[Pure, Actor] = Stream.chunk(Chunk.array(Array(
+ scarlettJohansson,
+ robertDowneyJr,
+ chrisEvans,
+ markRuffalo,
+ chrisHemsworth,
+ jeremyRenner
+)))
+```
+
+The fs2 library defines a lot of smart-constructors for the `Chunk` type, letting us create a `Chunk` from an `Option`, a `Seq`, a `Queue`, and so on.
+
+Most of the functions defined on streams are `Chunk`- aware, so we don't have to worry about chunks while working with them.
+
+## 3. Transforming a Stream
+
+Once we've built a Stream, we can transform its values more or less as we make in regular Scala collections. For example, let's create a stream containing all the actors whose hero belongs from the Justice League or the Avengers. We can use the `++` operator, which concatenates two streams:
+
+```scala
+val dcAndMarvelSuperheroes: Stream[Pure, Actor] = jlActors ++ avengersActors
+```
+
+So, the `dcAndMarvelSuperheroes` stream will emit all the actors from the Justice League and then the Avengers.
+
+**The `Stream` type forms a monad on the `O` type parameter**, which means that a `flatMap` method is available on streams, and we can use it to concatenate operations concerning the output values of the stream.
+
+For example, printing to the console the elements of a stream uses the `flatMap` method:
+
+```scala
+val printedJlActors: Stream[IO, Unit] = jlActors.flatMap { actor =>
+ Stream.eval(IO.println(actor))
+}
+```
+
+The pattern of calling the function `Stream.eval` inside a `flatMap` is so common that fs2 provides a shortcut for it through the `evalMap` method:
+
+```scala
+val evalMappedJlActors: Stream[IO, Unit] = jlActors.evalMap(IO.println)
+```
+
+If we need to perform some effects on the stream, but we don't want to change the type of the stream, we can use the `evalTap` method:
+
+```scala
+val evalTappedJlActors: Stream[IO, Actor] = jlActors.evalTap(IO.println)
+```
+
+As we can notice, the difference is that the `evalMap` function mapped the stream to a `Unit`, whereas the `evalTap` didn't perform any mapping.
+
+An essential feature of fs2 streams is that their functions take constant time, regardless of the structure of the stream itself. So, concatenating two streams is a constant time operation, whether the streams contain many elements or are infinite. As we will see in the rest of the article, this feature concerns the internal representation, which is implemented as a _pull-based_ structure.
+
+Since the functions defined on streams are many, we will make only a few examples. We want to group the Avengers by the actor's name playing them. We can do it using the `fold` method:
+
+```scala
+val avengersActorsByFirstName: Stream[Pure, Map[String, List[Actor]]] = avengersActors.fold(Map.empty[String, List[Actor]]) { (map, actor) =>
+ map + (actor.firstName -> (actor :: map.getOrElse(actor.firstName, Nil)))
+}
+```
+
+As we can see, the `fold` method works exactly as its Scala counterpart.
+
+**Many other streaming libraries define streams and transformation in terms of _sources_, _pipes_, and _sinks_**. A source generates the elements of a stream, then transformed through a sequence of stages or pipes, and finally consumed by a sink. For example, the Akka Stream library has specific types modeling these concepts.
+
+In fs2, the only available type modeling the above streaming concepts is `Pipe[F[_], -I, +O]`. `Pipe` is a type alias for the function `Stream[F, I] => Stream[F, O]`. So, a `Pipe[F[_], I, O]` represents nothing more than a function between two streams, the first emitting elements of type `I`, and the second emitting elements of type `O`.
+
+Using pipes, we can look at the definition of fs2 streams, like the definitions of streams in other libraries, representing transformations on streams as a sequence of stages. The `through` method applies a pipe to a stream. Its internal implementation is straightforward since it applies the function defined by the pipe to the stream:
+
+```scala
+// fs2 library code
+def through[F2[x] >: F[x], O2](f: Stream[F, O] => Stream[F2, O2]): Stream[F2, O2] = f(this)
+```
+
+For example, let's transform the `jlActors` streams into a stream containing only the first names of the actors and finally print them to the console:
+
+```scala
+val fromActorToStringPipe: Pipe[IO, Actor, String] = in =>
+ in.map(actor => s"${actor.firstName} ${actor.lastName}")
+def toConsole[T]: Pipe[IO, T, Unit] = in =>
+ in.evalMap(str => IO.println(str))
+val stringNamesOfJlActors: Stream[IO, Unit] =
+ jlActors.through(fromActorToStringPipe).through(toConsole)
+```
+
+The `jlActors` stream represents the _source_, whereas the `fromActorToStringPipe` represents a pipe, and the `toConsole` represents the _sink_. We can conclude that a pipe is pretty much a _map_/_flatMap_ type functional operation, but the pipe concept fits nicely into the mental model of a Stream.
+
+## 4. Error Handling in Streams
+
+What if pulling a value from a stream fails with an exception? For example, let's introduce a repository persisting an `Actor`:
+
+```scala
+object ActorRepository {
+ def save(actor: Actor): IO[Int] = IO {
+ println(s"Saving actor: $actor")
+ if (Random.nextInt() % 2 == 0) {
+ throw new RuntimeException("Something went wrong during the communication with the persistence layer")
+ }
+ println(s"Saved.")
+ actor.id
+ }
+}
+```
+
+The `save` method throws an exception when a randomly generated number is even, simulating a random error during the communication with the persistent layer. Then, we can use the above repository to persist a stream of actors:
+
+```scala
+val savedJlActors: Stream[IO, Int] = jlActors.evalMap(ActorRepository.save)
+```
+
+If we run the above stream, we will likely see the following output:
+
+```
+Saving actor: Actor(0,Henry,Cavill)
+java.lang.RuntimeException: Something went wrong during the communication with the persistence layer
+```
+
+As we can see, **the stream is interrupted by the exception**. In the above execution, it doesn't persist even the first actor. So, every time an exception occurs during pulling elements from a stream, the stream execution ends.
+
+However, fs2 gives us many choices to handle the error. First, we can handle an error returning a new stream using the `handleErrorWith` method:
+
+```scala
+val errorHandledSavedJlActors: Stream[IO, AnyVal] =
+ savedJlActors.handleErrorWith(error => Stream.eval(IO.println(s"Error: $error")))
+```
+
+In the above example, we react to an error by returning a stream that prints the error to the console. As we can notice, the elements contained in the stream are `AnyVal` and not `Unit` because of the definition of the `handleErrorWith`:
+
+```scala
+// fs2 library code
+def handleErrorWith[F2[x] >: F[x], O2 >: O](h: Throwable => Stream[F2, O2]): Stream[F2, O2] = ???
+```
+
+The `O2` type must be a supertype of `O`'s original type. Since both `Int` and `Unit` are subtypes of `AnyVal`, we can use the `AnyVal` type (the least common supertype) to represent the resulting stream.
+
+Another available method to handle errors in streams is `attempt`. The method works using the `scala.Either` type, which returns a stream of `Either` elements. The resulting stream pulls elements wrapped in a `Right` instance until the first error occurs, which is nothing more than an instance of a `Throwable` wrapped in a `Left`:
+
+```scala
+val attemptedSavedJlActors: Stream[IO, Either[Throwable, Int]] = savedJlActors.attempt
+attemptedSavedJlActors.evalMap {
+ case Left(error) => IO.println(s"Error: $error")
+ case Right(id) => IO.println(s"Saved actor with id: $id")
+}
+```
+
+For the sake of completeness, let's say that the `attempt` method is implemented internally using the `handleErrorWith` most straightforwardly:
+
+```scala
+// fs2 library code
+def attempt: Stream[F, Either[Throwable, O]] =
+ map(Right(_): Either[Throwable, O]).handleErrorWith(e => Stream.emit(Left(e)))
+```
+
+## 5. Resource Management
+
+As the official documentation says, we don't have to use the `handleErrorWith` method to manage the release of resources used by the stream eventually. Instead, **the fs2 library implements the _bracket_ pattern to manage resources**.
+
+The bracket pattern is a resource management pattern developed in the functional programming domain many years ago. The pattern defines two functions: The first is used to acquire a resource; The second is guaranteed to be called when the resource is no longer needed.
+
+The fs2 library implements the bracket pattern through the `bracket` method:
+
+```scala
+// fs2 library code
+def bracket[F[x] >: Pure[x], R](acquire: F[R])(release: R => F[Unit]): Stream[F, R] = ???
+```
+
+The function `acquire` is used to acquire a resource, and the function `release` is used to release the resource. The resulting stream has elements of type `R`. Moreover, **the resource is released at the end, both in case of success and in case of error**. Notice that both the `acquire` and `release` functions returns an effect since they can throw exceptions during the acquisition or release of resources.
+
+Let's make an example. We want to use a resource during the persistence of the stream containing the actors of the JLA. First, we define a value class representing a connection to a database:
+
+```scala
+case class DatabaseConnection(connection: String) extends AnyVal
+```
+
+We will use the `DatabaseConnection` as the resource we want to acquire and release through the bracket pattern. Then, the acquiring and releasing function:
+
+```scala
+val acquire = IO {
+ val conn = DatabaseConnection("jlaConnection")
+ println(s"Acquiring connection to the database: $conn")
+ conn
+}
+
+val release = (conn: DatabaseConnection) =>
+ IO.println(s"Releasing connection to the database: $conn")
+```
+
+Finally, we use them to call the `bracket` method and then save the actors in the stream:
+
+```scala
+val managedJlActors: Stream[IO, Int] =
+ Stream.bracket(acquire)(release).flatMap(conn => savedJlActors)
+```
+
+Since the `savedJlActors` we defined some time ago throws an exception when the stream is evaluated, the `managedJlActors` stream will terminate with an error. The `release` function is called, and the `conn` value is released. The execution output of the `managedJlActors` stream should be something similar to the following:
+
+```
+Acquiring connection to the database: DatabaseConnection(jlaConnection)
+Saving actor: Actor(0,Henry,Cavill)
+Releasing connection to the database: DatabaseConnection(jlaConnection)
+java.lang.RuntimeException: Something went wrong during the communication with the persistence layer
+```
+
+As we can see, the resource we created was released, even if the stream had terminated with an error. That's awesome!
+
+## 6. The `Pull` Type
+
+As we said more than once, the fs2 defines streams as a _pull_ type, which means that the **stream effectively computes the next stream element just in time**.
+
+Under the hood, the library implements the `Stream` type functions using the `Pull` type to honor this behavior. This type, also available as a public API, lets us implement streams using the pull model.
+
+The `Pull[F[_], O, R]` type represents a program that can pull output values of type `O` while computing a result of type `R` while using an effect of type `F`. As we can see, the type introduces the new type variable `R` that is not available in the `Stream` type.
+
+The result `R` represents the information available after the emission of the element of type `O` that should be used to emit the next value of a stream. For this reason, using `Pull` directly means to develop recursive programs. Ok, one step at a time. Let's analyze the `Pull` type and its methods first.
+
+**The `Pull` type represents a stream as a _head_ and a _tail_**, much like we can describe a list. The element of type `O` emitted by the `Pull` represents the head. However, since a stream is a possible infinite data structure, we cannot express it with a finite one. So, we return a type `R`, which is all the information that we need to compute the tail of the stream.
+
+Let's look at the `Pull` type methods without further ado. The first method we encounter is the smart constructor `output1`, which creates a `Pull` that emits a single value of type `O` and then completes.
+
+```scala
+val tomHollandActorPull: Pull[Pure, Actor, Unit] = Pull.output1(tomHolland)
+```
+
+We can convert a `Pull` having the `R` type variable bound to `Unit` directly to a `Stream` by using the `stream` method:
+
+```scala
+val tomHollandActorStream: Stream[Pure, Actor] = tomHollandActorPull.stream
+```
+
+Revamping the analogy with the finite collection, a `Pull` that returns `Unit` is like a `List` with a head and empty tail.
+
+Unlike the `Stream` type, which defines a monad instance on the type variable `O`, a `Pull` forms a monad instance on `R`. If we think, it's logical: All we want is to concatenate the information that allows us to compute the tail of the stream. So, if we want to create a sequence of `Pull` containing all the actors that play Spider-Man, we can do the following:
+
+```scala
+val spiderMenActorPull: Pull[Pure, Actor, Unit] =
+ tomHollandActorPull >> Pull.output1(tobeyMaguire) >> Pull.output1(andrewGarfield)
+```
+
+Conversely, we can convert a `Stream` into a `Pull` using the `echo` method:
+
+```scala
+val avengersActorsPull: Pull[Pure, Actor, Unit] = avengersActors.pull.echo
+```
+
+In the above example, the first invoked function is `pull`, which returns a `ToPull[F, O]` type. This last type is a wrapper around the `Stream` type, which aim is to group all functions concerning the conversion into a `Pull` instance:
+
+```scala
+// fs2 library code
+final class ToPull[F[_], O] private[Stream] (
+ private val self: Stream[F, O]) extends AnyVal
+```
+
+Then, the `echo` function makes the conversion. It's interesting to look at the implementation of the `echo` method since it makes no conversion at all:
+
+```scala
+// fs2 library code
+def echo: Pull[F, O, Unit] = self.underlying
+```
+
+The `echo` function returns the internal representation of the `Stream`, called `underlying` since a stream is represented as a pull internally:
+
+```scala
+// fs2 library code
+final class Stream[+F[_], +O] private[fs2] (private[fs2] val underlying: Pull[F, O, Unit])
+```
+
+Another way to convert a `Stream` into a `Pull` is to use the `uncons` function, which returns a `Pull` pulling a tuple containing the head chunk of the stream and its tail. For example, we can call `uncons` the `avengersActors` stream:
+
+```scala
+val unconsAvengersActors: Pull[Pure, INothing, Option[(Chunk[Actor], Stream[Pure, Actor])]] =
+ avengersActors.pull.uncons
+```
+
+Wow! It's a very intricate type. Let's describe it step by step. First, since the original stream uses the `Pure` effect, the resulting `Pull` also uses the same effect. Then, since the `Pull` deconstructs the original stream, it cannot emit any value, and so the output type is `INothing` (type alias for scala `Nothing`). It follows the value returned by the `Pull`, i.e., the deconstruction of the original `Stream`.
+
+The returned value is an `Option` because the `Stream` may be empty: If there is no more value in the original `Stream`, we will have a `None`. Otherwise, we will have the head of the stream as a `Chunk` and a `Stream` representing the tail of the original stream.
+
+A variant of the original `uncons` method returns not the first `Chunk` but the first stream element. It's called `uncons1`:
+
+```scala
+val uncons1AvengersActors: Pull[Pure, INothing, Option[(Actor, Stream[Pure, Actor])]] =
+ avengersActors.pull.uncons1
+```
+
+With these bullets in our gun, it's time to write down some functions that use the `Pull` type. Due to the structure of the type, the functions implemented using the `Pull` type are often recursive.
+
+For example, without using the `Stream.filter` method, we can write a pipe filtering from a stream of actors, all of them with a given first name:
+
+```scala
+def takeByName(name: String): Pipe[IO, Actor, Actor] =
+ def go(s: Stream[IO, Actor], name: String): Pull[IO, Actor, Unit] =
+ s.pull.uncons1.flatMap {
+ case Some((hd, tl)) =>
+ if (hd.firstName == name) Pull.output1(hd) >> go(tl, name)
+ else go(tl, name)
+ case None => Pull.done
+ }
+ in => go(in, name).stream
+```
+
+First, we need to accumulate the actors in the stream that fulfill the filtering condition. So, we apply a typical functional programming pattern and define an inner function that we use to recur. Let's call this function `go`.
+
+Then, we deconstruct the stream using `uncons1` to retrieve its first element. Since the `Pull` type forms a monad on the `R` type, we can use the `flatMap` method to recur:
+
+```scala
+s.pull.uncons1.flatMap {
+```
+If we have a `None` value, then we're done, and terminate the recursion using the `Pull.done` method, returning a `Pull[Pure, INothing, Unit]` instance:
+```scala
+case None => Pull.done
+```
+Otherwise, we pattern-match the `Some` value:
+```scala
+case Some((hd, tl)) =>
+```
+If the actor's first name representing the head of the stream has the right first name, we output the actor and recur with the tail of the stream. Otherwise, we recur with the tail of the stream:
+```scala
+if (hd.firstName == name) Pull.output1(hd) >> go(tl, name)
+else go(tl, name)
+```
+Finally, we define the whole `Pipe` calling the `go` function and use the `stream` method to convert the `Pull` instance into a `Stream` instance:
+```scala
+in => go(in, name).stream
+```
+Now, we can use the pipe we just defined to filter the avengers' stream, getting only the actors with the string "Chris" as the first name:
+```scala
+val avengersActorsCalledChris: Stream[IO, Unit] =
+ avengersActors.through(takeByName("Chris")).through(toConsole)
+```
+## 7. Using Concurrency in Streams
+One of the most used fs2 features is using concurrency in streams. It's possible to implement many concurrency patterns using the primitives provided by the `Stream` type.
+The first function we will analyze is `merge`, which lets us run two streams concurrently, collecting the results in a single stream as they are produced. **The execution halts when both streams have halted**. For example, we can merge JLA and Avengers concurrently, adding some sleep to the execution of each stream:
+```scala
+val concurrentJlActors: Stream[IO, Actor] = liftedJlActors.evalMap(actor => IO {
+ Thread.sleep(400)
+ actor
+})
+val liftedAvengersActors: Stream[IO, Actor] = avengersActors.covary[IO]
+val concurrentAvengersActors: Stream[IO, Actor] = liftedAvengersActors.evalMap(actor => IO {
+ Thread.sleep(200)
+ actor
+})
+val mergedHeroesActors: Stream[IO, Unit] =
+ concurrentJlActors.merge(concurrentAvengersActors).through(toConsole)
+```
+The output to the console of the above program is something similar to the following:
+```
+Actor(7,Scarlett,Johansson)
+Actor(0,Henry,Cavill)
+Actor(8,Robert,Downey Jr.)
+Actor(9,Chris,Evans)
+Actor(1,Gal,Godot)
+Actor(10,Mark,Ruffalo)
+Actor(11,Chris,Hemsworth)
+Actor(2,Ezra,Miller)
+Actor(12,Jeremy,Renner)
+Actor(3,Ben,Fisher)
+Actor(4,Ray,Hardy)
+Actor(5,Jason,Momoa)
+```
+As we may expect, the stream contains an actor of the JLA more or less every two actors of the Avengers. Once the Avengers actors are finished, the JLA actors fulfill the rest of the stream.
+If we don't care about the results of the second stream running concurrently, we can use the `concurrently` method instead. An everyday use case for this method is implementing a producer-consumer pattern. For example, we can implement a program with a producer that uses a `cats.effect.std.Queue` to share JLA actors with a consumer, which prints them to the console:
+```scala
+val queue: IO[Queue[IO, Actor]] = Queue.bounded[IO, Actor](10)
+val concurrentlyStreams: Stream[IO, Unit] = Stream.eval(queue).flatMap { q =>
+ val producer: Stream[IO, Unit] =
+ liftedJlActors
+ .evalTap(actor => IO.println(s"[${Thread.currentThread().getName}] produced $actor"))
+ .evalMap(q.offer)
+ .metered(1.second)
+ val consumer: Stream[IO, Unit] =
+ Stream.fromQueueUnterminated(q)
+ .evalMap(actor => IO.println(s"[${Thread.currentThread().getName}] consumed $actor"))
+ producer.concurrently(consumer)
+}
+```
+The first line declares a `Queue` of `Actor` of size ten. Then, we use streams to transform the effect containing the queue. In detail, we declared a `producer` stream, which scans the JLA actors and adds them to the queue. We also add a print statement to the console showing the thread name and the actor:
+```scala
+val producer: Stream[IO, Unit] =
+ liftedJlActors
+ .evalTap(actor => IO.println(s"[${Thread.currentThread().getName}] produced
+ .evalMap(q.offer)
+ .metered(1.second)
+```
+The `metered` method allows waiting a given time between two consecutive pulls. We decided to wait one second.
+Then, we declare a `consumer` stream, which pulls an actor from the queue and prints it to the console with the thread name:
+```scala
+val consumer: Stream[IO, Unit] =
+ Stream.fromQueueUnterminated(q)
+ .evalMap(actor => IO.println(s"[${Thread.currentThread().getName}] consumed $actor"))
+```
+Finally, we run both the producer and the consumer concurrently:
+```scala
+producer.concurrently(consumer)
+```
+Running the above program produces an output similar to the following:
+```
+[io-compute-2] produced Actor(0,Henry,Cavill)
+[io-compute-3] consumed Actor(0,Henry,Cavill)
+[io-compute-2] produced Actor(1,Gal,Godot)
+[io-compute-0] consumed Actor(1,Gal,Godot)
+[io-compute-0] produced Actor(2,Ezra,Miller)
+[io-compute-3] consumed Actor(2,Ezra,Miller)
+[io-compute-1] produced Actor(3,Ben,Fisher)
+[io-compute-3] consumed Actor(3,Ben,Fisher)
+[io-compute-1] produced Actor(4,Ray,Hardy)
+[io-compute-0] consumed Actor(4,Ray,Hardy)
+[io-compute-1] produced Actor(5,Jason,Momoa)
+[io-compute-2] consumed Actor(5,Jason,Momoa)
+```
+As we can see, the producer and consumer are indeed running concurrently.
+A critical feature of running two streams through the `concurrently` method is that **the second stream halts when the first stream is finished**.
+Moreover, we can run a set of streams concurrently, deciding the degree of concurrency _a priori_ using streams. The method `parJoin` does precisely this. Contrary to the `concurrently` method, the results of the streams are merged in a single stream, and no assumption is made on streams' termination.
+We can try to print the actors playing superheroes of the JLA, the Avengers, and Spider-Man concurrently. First, we define a `Pipe` printing the thread name and the actor being processed:
+```scala
+val toConsoleWithThread: Pipe[IO, Actor, Unit] = in =>
+ in.evalMap(actor => IO.println(s"[${Thread.currentThread().getName}] consumed $actor"))
+```
+Then, we can define the stream using the `parJoin` method:
+```scala
+val parJoinedActors: Stream[IO, Unit] =
+ Stream(
+ jlActors.through(toConsoleWithThread),
+ avengersActors.through(toConsoleWithThread),
+ spiderMen.through(toConsoleWithThread)
+ ).parJoin(4)
+```
+The method is available only on streams of type `Stream[F, Stream[F, O]]`. It's not part directly of the `Stream` API, but an implicit conversion adds it as an _extension method_:
+```scala
+// fs2 library code
+implicit final class NestedStreamOps[F[_], O](private val outer: Stream[F, Stream[F, O]])
+ extends AnyVal {
+ def parJoin(maxOpen: Int)(implicit F: Concurrent[F]): Stream[F, O] = ???
+}
+```
+Each stream is traversed concurrently, and the maximum degree of concurrency is given in input as the parameter `maxOpen`. **As we can see, the effect used to handle concurrency must satisfy the `Concurrent` bound, which is required anywhere concurrency is used in the library**.
+We want four threads to pull concurrent values from the three streams in our example. Running the program produces an output similar to the following:
+```
+[io-compute-3] consumed Actor(7,Scarlett,Johansson)
+[io-compute-1] consumed Actor(0,Henry,Cavill)
+[io-compute-0] consumed Actor(13,Tom,Holland)
+[io-compute-2] consumed Actor(8,Robert,Downey Jr.)
+[io-compute-1] consumed Actor(1,Gal,Godot)
+[io-compute-2] consumed Actor(9,Chris,Evans)
+[io-compute-1] consumed Actor(14,Tobey,Maguire)
+[io-compute-1] consumed Actor(10,Mark,Ruffalo)
+[io-compute-2] consumed Actor(2,Ezra,Miller)
+[io-compute-2] consumed Actor(15,Andrew,Garfield)
+[io-compute-0] consumed Actor(3,Ben,Fisher)
+[io-compute-1] consumed Actor(11,Chris,Hemsworth)
+[io-compute-3] consumed Actor(12,Jeremy,Renner)
+[io-compute-0] consumed Actor(4,Ray,Hardy)
+[io-compute-2] consumed Actor(5,Jason,Momoa)
+```
+As we can see, the threads used by the runtime are precisely four.
+Many other methods are available in the fs2 library concerning concurrency, such as `either` and `mergeHaltBoth`. Please, refer to the [documentation](https://oss.sonatype.org/service/local/repositories/releases/archive/co/fs2/fs2-core_2.13/3.2.0/fs2-core_2.13-3.2.0-javadoc.jar/!/fs2/index.html) for more details.
+
+## 8. Conclusion
+Our long journey through the main features of the fs2 library comes to an end. During the path, we introduced the concepts of `Stream` and how to declare them, both using pure values and in an effectful way. We analyzed streams internals, such as `Chunk`s and `Pull` types. We saw how to handle errors and how a stream can safely acquire and release a resource. Finally, we focused on concurrency and how streams can deal with concurrent programming.
+
+However, fs2 is so much more. We didn't speak about how to interact with the external world, for example, reading and writing files. We didn't see how to interrupt the execution of a stream. Moreover, to mention some, we didn't cite the many libraries built on top of fs2, such as _doobie_ or _http4s_. However, the [documentation](https://fs2.io/#/) of fs2 is very comprehensive, and we can refer to it for more details.
diff --git a/_posts/2022-03-18-snake.md b/_posts/2022-03-18-snake.md
new file mode 100644
index 000000000000..0d329cd53ba9
--- /dev/null
+++ b/_posts/2022-03-18-snake.md
@@ -0,0 +1,482 @@
+---
+title: "How to make a Snake game in Scala"
+date: 2022-02-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scalafx, game]
+excerpt: "The ultimate 10-minute tutorial to a Snake game in Scala."
+---
+
+This article is for fun. We're going to learn how to make a Snake game in Scala and ScalaFX in just a few minutes.
+
+This game was previously posted as a [video](https://www.youtube.com/watch?v=sp6QO6eRRrg). In there, I also wanted to beat a psychological time barrier (10 mins) and implement the game from (almost) scratch.
+
+Check the video below, if you want more "action":
+
+{% include video id="sp6QO6eRRrg" provider="youtube" %}
+
+I also talk about some of the logical steps to creating this game on camera, but due to the time pressure, some details were skipped. This article wants to talk about the thought process step by step.
+
+## Intro
+
+We're going to use [ScalaFX](https://scalafx.org), a wrapper library over [JavaFX](https://openjfx.io/) for GUIs, with a few Scala niceties. The library is not really functional-first, but it does have some expressiveness to it.
+
+In order to add ScalaFX to our project, we're going to add their default `build.sbt` as follows:
+
+```scala
+scalaVersion := "2.13.8"
+
+// Add dependency on ScalaFX library
+libraryDependencies += "org.scalafx" %% "scalafx" % "16.0.0-R25"
+
+// Determine OS version of JavaFX binaries
+lazy val osName = System.getProperty("os.name") match {
+ case n if n.startsWith("Linux") => "linux"
+ case n if n.startsWith("Mac") => "mac"
+ case n if n.startsWith("Windows") => "win"
+ case _ => throw new Exception("Unknown platform!")
+}
+
+// Add dependency on JavaFX libraries, OS dependent
+lazy val javaFXModules = Seq("base", "controls", "fxml", "graphics", "media", "swing", "web")
+libraryDependencies ++= javaFXModules.map(m =>
+ "org.openjfx" % s"javafx-$m" % "16" classifier osName
+)
+```
+
+With the `build.sbt` file prepared, we'll need to add some minor boilerplate to have a simple ScalaFX application which opens a window filled with white:
+
+```scala
+// all the imports we're going to need in the entire app
+// (auto-import helps a lot, but adding them here in case of confusion)
+import scalafx.application.{JFXApp3, Platform}
+import scalafx.beans.property.{IntegerProperty, ObjectProperty}
+import scalafx.scene.Scene
+import scalafx.scene.paint.Color
+import scalafx.scene.paint.Color._
+import scalafx.scene.shape.Rectangle
+
+import scala.concurrent.Future
+import scala.util.Random
+
+object SnakeFx extends JFXApp3 {
+ override def start(): Unit = {
+ stage = new JFXApp3.PrimaryStage {
+ width = 600
+ height = 600
+ scene = new Scene {
+ fill = White
+ }
+ }
+ }
+}
+```
+
+## Drawing
+
+In order to draw something on screen, we need to modify the `content` field of the `scene` field of the `stage` field of the main application. That was a lot of indirection. More concretely, in order to draw a green rectangle of length 25 at coordinates (50, 75), we would write something like this:
+
+```scala
+stage = new JFXApp3.PrimaryStage {
+ width = 600
+ height = 600
+ scene = new Scene {
+ fill = White
+ // added just now
+ content = new Rectangle {
+ x = 50
+ y = 75
+ width = 25
+ height = 25
+ fill = Green
+ }
+ }
+}
+```
+
+And we'd get something fabulous:
+
+
+
+Coordinates start from the top left; the x coordinate goes to the right, the y coordinate goes down.
+
+Drawing a rectangle is so useful, that we'll take the Rectangle expression and call it from a method:
+
+```scala
+ def square(xr: Double, yr: Double, color: Color) = new Rectangle {
+ x = xr
+ y = yr
+ width = 25
+ height = 25
+ fill = color
+ }
+```
+
+For the simplicity of this game, we'll consider the snake built from squares of equal size and color green (it's supposed to be a snake), and the food as a red square that is generated randomly on sceen every time the snake eats it.
+
+Now, for the logic.
+
+## Logic
+
+All we need for the Snake game is to draw rectangles on the screen. The question is, where?
+
+For the logic of the game, we'll consider the snake as a list of `(x,y)` coordinates, which we'll then use to draw squares with our magical `square` function. Remember that `content` field in the scene? That can also be a collection of graphics, not just a single one — so we can safely use our list of squares as the proper value.
+
+So let's start with an initial set of coordinates for the snake: let's assume a 3-square snake of the form
+
+```scala
+ val initialSnake: List[(Double, Double)] = List(
+ (250, 200),
+ (225, 200),
+ (200, 200)
+ )
+```
+
+and consider the state of the game as a data structure of the form
+
+```scala
+case class State(snake: List[(Double, Double)], food: (Double, Double))
+```
+
+The game is deterministic. Given a set direction, we know where the snake is going to go. So we can safely update the state into a new state given a direction. Adding a method to the `State` case class:
+
+```scala
+def newState(dir: Int): State = ???
+```
+
+Inside the `newState` method, we have to do the following:
+
+- update the new head of the snake, given the direction
+- update the rest of the snake by placing the _last_ n-1 squares at the positions of the _first_ n-1 squares
+- check if we're out of the scene boundaries OR eating our own tail; reset the state in this case
+- check if the snake is just eating the food; re-generate food coordinates in that case
+
+Rock'n roll. Updating the snake's head needs to take into account the direction; consider directions 1,2,3,4 as up, down, left, right:
+
+```scala
+ val (x, y) = snake.head
+ val (newx, newy) = dir match {
+ case 1 => (x, y - 25) // up
+ case 2 => (x, y + 25) // down
+ case 3 => (x - 25, y) // left
+ case 4 => (x + 25, y) // right
+ case _ => (x, y)
+ }
+```
+
+Crashing into the scene boundaries means `newx < 0 || newx >= 600 || newy < 0 || newy >= 600` (with some additional constants instead of 600 if you don't want anything hardcoded). Eating the snake's tail literally means that `snake.tail` contains a tuple equal to the newly created one.
+
+```scala
+ val newSnake: List[(Double, Double)] =
+ if (newx < 0 || newx >= 600 || newy < 0 || newy >= 600 || snake.tail.contains((newx, newy)))
+ initialSnake
+ else ???
+```
+
+Otherwise, eating the food means that the new tuple is on the same coordinates as the food, so we need to prepend a new item to the snake list:
+
+```scala
+// (keeping the above)
+else if (food == (newx, newy))
+ food :: snake
+else ???
+```
+
+Otherwise, we need to keep moving the snake. The new head was already computed as `(newx, newy)`, so we need to bring the rest of the snake:
+
+```scala
+// keeping the above
+else (newx, newy) :: snake.init
+```
+
+Using `snake.init` as the coordinates of the _first_ n-1 pieces of the snake. With the new head at the front, that comprises the new snake with the same length as before. The `init` method is really cool in this case.
+
+To return a new `State` instance, we also need to update the coordinates of the food if it was just eaten. That said:
+
+```scala
+ val newFood =
+ if (food == (newx, newy))
+ randomFood()
+ else
+ food
+```
+
+Where `randomFood` is a method to create a random square somewhere on the stage:
+
+```scala
+ def randomFood(): (Double, Double) =
+ (Random.nextInt(24) * 25 , Random.nextInt(24) * 25)
+```
+
+If you want to create a stage of different size, say `L x h`, you'd have
+
+```scala
+ def randomFood(): (Double, Double) =
+ (Random.nextInt(L / 25) * 25 , Random.nextInt(h / 25) * 25)
+```
+
+Back to the `newState` method. Given that we've just determined the new snake and the new food, all we need is to return `State(newSnake, newFood)`, which brings the main state update function to:
+
+```scala
+def newState(dir: Int): State = {
+ val (x, y) = snake.head
+ val (newx, newy) = dir match {
+ case 1 => (x, y - 25)
+ case 2 => (x, y + 25)
+ case 3 => (x - 25, y)
+ case 4 => (x + 25, y)
+ case _ => (x, y)
+ }
+
+ val newSnake: List[(Double, Double)] =
+ if (newx < 0 || newx >= 600 || newy < 0 || newy >= 600 || snake.tail.contains((newx, newy)))
+ initialSnake
+ else if (food == (newx, newy))
+ food :: snake
+ else
+ (newx, newy) :: snake.init
+
+ val newFood =
+ if (food == (newx, newy))
+ randomFood()
+ else
+ food
+
+ State(newSnake, newFood)
+}
+```
+
+Then what? We'll need to be able to display that state on the screen, so we'll need a method to turn a State into a bunch of squares, so adding another method in `State` which turns `food` into a red square and all the pieces of the snake into green squares:
+
+```scala
+// inside the State class
+def rectangles: List[Rectangle] = square(food._1, food._2, Red) :: snake.map {
+ case (x, y) => square(x,y, Green)
+}
+```
+
+## Plugging Snake Logic Into ScalaFX
+
+Our pure game logic is complete, we just need to be able to use this state somewhere, and run a game loop or update function constantly and re-draw things on the stage. To that end, we're going to create 3 ScalaFX "properties" which are essentially glorified variables with `onChange` listeners:
+
+- a property that describes the current state of the game as an instance of `State`
+- a property that keeps track of the current direction, changeable by key presses
+- a property that keeps the current frame, updating every X milliseconds
+
+At the beginning of the main app's `start()` method, we're going to add the following:
+
+```scala
+ val state = ObjectProperty(State(initialSnake, randomFood()))
+ val frame = IntegerProperty(0)
+ val direction = IntegerProperty(4) // 4 = right
+```
+
+We know that on every frame change, we have to update the state, considering the current value of `direction`, so we're going to add
+
+```scala
+frame.onChange {
+ state.update(state.value.newState(direction.value))
+}
+```
+
+So the state will update automatically once the frame changes. So we need to make sure of 3 things:
+
+- to draw the current state's squares on the screen
+- to update the direction variable based on key presses
+- to change/increase the frame number every X milliseconds (picked 80 or 100 for smooth game play)
+
+Piece 1 is easy. We need to change the `content` field of the scene to be
+
+```scala
+content = state.value.rectangles
+```
+
+Even having the app as it is, we can run it to validate that we have a snake and food on screen:
+
+
+
+Obviously, nothing changes because the frame doesn't change. If the frame changes, so will the state. If the state ever changes, so will the content. Still inside the `Scene` constructor, we need to be able to update this content when the state changes:
+
+```scala
+// complete scene field of the stage
+scene = new Scene {
+ fill = White
+ content = state.value.rectangles
+ state.onChange {
+ content = state.value.rectangles
+ }
+}
+```
+
+That was the first bullet: drawing the state's squares on the screen. The second bullet is to update the direction based on key presses. Thankfully, there is a key press listener directly in the scene, so the scene will now look like this:
+
+```scala
+stage = new JFXApp3.PrimaryStage {
+ width = 600
+ height = 600
+ scene = new Scene {
+ fill = White
+ content = state.value.rectangles
+ // added now
+ onKeyPressed = key => key.getText match {
+ case "w" => direction.value = 1
+ case "s" => direction.value = 2
+ case "a" => direction.value = 3
+ case "d" => direction.value = 4
+ }
+
+ state.onChange {
+ content = state.value.rectangles
+ }
+ }
+}
+```
+
+Again, if we run the app, all is static because nothing triggers any state change. We'll need to update the frame, which will trigger everything.
+
+The problem with the frame update is that we cannot block the main display thread. So we'll need to do it from another thread. We'll define a general game loop that runs any sort of function, waits 80 millis or so and then runs the function again. Of course, asynchronously.
+
+```scala
+
+import scala.concurrent.ExecutionContext.Implicits.global
+
+def gameLoop(update: () => Unit): Unit =
+ Future {
+ update()
+ Thread.sleep(80)
+ }.flatMap(_ => Future(gameLoop(update)))
+```
+
+Now, all we need to do is trigger this game loop with a function that changes the frame. Changing of frame triggers changing of state, changing of state triggers new display. That's the idea at least. At the bottom of the app's `start()` method, we'll add
+
+```scala
+gameLoop(() => frame.update(frame.value + 1))
+```
+
+Running it, _we get into an error_ because we're blocking the main display thread when we're updating the `content`, so we need instead to _schedule_ that update by replacing
+
+```scala
+state.onChange {
+ content = state.value.rectangles
+}
+```
+
+with
+
+```scala
+state.onChange(Platform.runLater {
+ content = state.value.rectangles
+})
+```
+
+which puts that display update in a queue of actions that the main display thread is supposed to execute.
+
+## Conclusion
+
+That's it, folks - a fully functional Snake game in Scala with ScalaFX in just a few minutes. The complete code is below:
+
+```scala
+
+import scalafx.application.{JFXApp3, Platform}
+import scalafx.beans.property.{IntegerProperty, ObjectProperty}
+import scalafx.scene.Scene
+import scalafx.scene.paint.Color
+import scalafx.scene.paint.Color._
+import scalafx.scene.shape.Rectangle
+
+import scala.concurrent.Future
+import scala.util.Random
+
+object SnakeFx extends JFXApp3 {
+
+ val initialSnake: List[(Double, Double)] = List(
+ (250, 200),
+ (225, 200),
+ (200, 200)
+ )
+
+ import scala.concurrent.ExecutionContext.Implicits.global
+
+ def gameLoop(update: () => Unit): Unit =
+ Future {
+ update()
+ Thread.sleep(1000 / 25 * 2)
+ }.flatMap(_ => Future(gameLoop(update)))
+
+ case class State(snake: List[(Double, Double)], food: (Double, Double)) {
+ def newState(dir: Int): State = {
+ val (x, y) = snake.head
+ val (newx, newy) = dir match {
+ case 1 => (x, y - 25)
+ case 2 => (x, y + 25)
+ case 3 => (x - 25, y)
+ case 4 => (x + 25, y)
+ case _ => (x, y)
+ }
+
+ val newSnake: List[(Double, Double)] =
+ if (newx < 0 || newx >= 600 || newy < 0 || newy >= 600 || snake.tail.contains((newx, newy)))
+ initialSnake
+ else if (food == (newx, newy))
+ food :: snake
+ else
+ (newx, newy) :: snake.init
+
+ val newFood =
+ if (food == (newx, newy))
+ randomFood()
+ else
+ food
+
+ State(newSnake, newFood)
+ }
+
+ def rectangles: List[Rectangle] = square(food._1, food._2, Red) :: snake.map {
+ case (x, y) => square(x, y, Green)
+ }
+ }
+
+ def randomFood(): (Double, Double) =
+ (Random.nextInt(24) * 25, Random.nextInt(24) * 25)
+
+ def square(xr: Double, yr: Double, color: Color) = new Rectangle {
+ x = xr
+ y = yr
+ width = 25
+ height = 25
+ fill = color
+ }
+
+ override def start(): Unit = {
+ val state = ObjectProperty(State(initialSnake, randomFood()))
+ val frame = IntegerProperty(0)
+ val direction = IntegerProperty(4) // right
+
+ frame.onChange {
+ state.update(state.value.newState(direction.value))
+ }
+
+ stage = new JFXApp3.PrimaryStage {
+ width = 600
+ height = 600
+ scene = new Scene {
+ fill = White
+ content = state.value.rectangles
+ onKeyPressed = key => key.getText match {
+ case "w" => direction.value = 1
+ case "s" => direction.value = 2
+ case "a" => direction.value = 3
+ case "d" => direction.value = 4
+ }
+
+ state.onChange(Platform.runLater {
+ content = state.value.rectangles
+ })
+ }
+ }
+
+ gameLoop(() => frame.update(frame.value + 1))
+ }
+
+}
+```
diff --git a/_posts/2022-03-22-akka-actor-discovery.md b/_posts/2022-03-22-akka-actor-discovery.md
new file mode 100644
index 000000000000..023629f10e12
--- /dev/null
+++ b/_posts/2022-03-22-akka-actor-discovery.md
@@ -0,0 +1,366 @@
+---
+title: "Akka Typed: Actor Discovery"
+date: 2022-03-22
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka]
+excerpt: "A common pattern in Akka Typed: how to find actors that are not explicitly passed around."
+---
+
+In this article, we're going to take a look at Akka Discovery, a feature that allows us to locate actors that we normally don't have access to, and that no other actors can notify us for.
+
+We talk about discovery in the [Akka Typed](https://rockthejvm.com/akka-essentials) course, in the context of routers and work distribution. It's a pretty powerful technique. This article assumes some basic familiarity with typed actors.
+
+If you want the video version, check below:
+
+{% include video id="iuq-JBUiLpI" provider="youtube" %}
+
+
+## 1. Introduction and Setup
+
+We'll use the regular Akka Typed Actors library, so in your Scala project, add the following to your `build.sbt`:
+
+```scala
+val AkkaVersion = "2.6.19"
+libraryDependencies ++= Seq(
+ "com.typesafe.akka" %% "akka-actor-typed" % AkkaVersion
+)
+```
+
+We assume some basics with Akka actors, but in a nutshell:
+
+- our application is organized in terms of independent, thread-safe entities called _actors_
+- we make our components/actors interact with each other by passing messages (asynchronously)
+- our actors are organized in a tree-like hierarchy, where each actor is responsible for its direct children
+- our actors can only receive a certain type of messages
+
+Having worked with actors, you've probably needed to pass a reference belonging to one part of the hierarchy, to another part of the hierarchy, so that these actors can communicate and exchange data. The trouble is that managing such exchanges needlessly complicates the logic of both hierarchies/departments and mixes in business logic with management logic.
+
+Akka Discovery is a feature that allows us to find actors that we cannot otherwise have access to. Its use cases include
+
+- the elimination of the above example scenario, allowing us to fully focus on business logic
+- locating an actor that we cannot possibly reach otherwise
+- notifications of actor reference changes
+
+For this article, we'll imagine a fictitious IOT application, where we have
+
+- a bunch of sensors as independent actors
+- a sensor controller in charge of the sensors, which sends heartbeat messages to the sensors
+- a data collector/aggregator, that receives data from the sensors
+- a guardian actor in charge of everyone
+
+Whenever the sensor controller sends the heartbeat message, the sensors must all send their readings to the data aggregator. At all times, the sensors must be running, but the data aggregator might change/be swapped in the meantime. Now, for some reason (logic too complicated, data privacy, impracticability etc), the guardian actor cannot directly communicate to the sensors to update them of the data aggregator change. We need to notify them in some other way.
+
+## 2. Creating the Actors
+
+Considering the following imports for the entire article
+
+```scala
+import akka.NotUsed
+import akka.actor.typed.receptionist.{Receptionist, ServiceKey}
+import akka.actor.typed.{ActorRef, ActorSystem, Behavior}
+import akka.actor.typed.scaladsl.Behaviors
+
+import scala.concurrent.duration._
+import scala.util.Random
+```
+
+we're going to sketch the main entities in our application:
+
+```scala
+
+// data aggregator domain
+case class SensorReading(id: String, value: Double)
+
+object DataAggregator {
+ def apply(): Behavior[SensorReading] = ???
+}
+
+// sensors domain
+trait SensorCommand
+case object SensorHeartbeat extends SensorCommand
+case class ChangeDataAggregator(agg: Option[ActorRef[SensorReading]]) extends SensorCommand
+object Sensor {
+ // the sensor actor
+ def apply(id: String): Behavior[SensorCommand] = ???
+
+ // the sensor aggregator
+ def controller(): Behavior[NotUsed] = ???
+}
+```
+
+while the main guardian would look something like this:
+
+```scala
+val guardian: Behavior[NotUsed] = Behaviors.setup { context =>
+ // controller for the sensors
+ context.spawn(Sensor.controller(), "controller")
+ val dataAgg1 = context.spawn(DataAggregator(), "data_agg_1")
+ // TODO make it "known" to the sensors that dataAgg1 is the new data aggregator
+ val dataAgg2 = context.spawn(DataAggregator(), "data_agg_2")
+ // TODO after 10 seconds, make it "known" to the sensors that dataAgg2 is the new data aggregator
+ Behaviors.empty
+}
+```
+
+and the main method
+
+```scala
+def main(args: Array[String]): Unit = {
+ val system = ActorSystem(guardian, "ActorDiscovery")
+ import system.executionContext
+ system.scheduler.scheduleOnce(20.seconds, () => system.terminate())
+}
+```
+
+The application should work as follows:
+
+- upon starting, the sensor controller would send heartbeats every second
+- with every heartbeat, the sensors would (ideally) send their sensor readings containing their id and the reading (a Double value) to the data aggregator
+- upon receiving a `SensorReading`, the data aggregator would display the latest data; in real life, this would be in a graph, or feeding in a data streaming application like Flink or Spark Streaming, but here we'll simply log the data
+- ideally, the sensors can "magically" find the data aggregator to send data to, published by the guardian actor
+- after 10 seconds of work, the guardian transparently swaps the data aggregator with a new one
+- the sensors will continue to work and push the data to the new aggregator, again, by magically being notified that the aggregator's location was changed
+- after 20 seconds, the entire application will be terminated
+
+The "magical" finding of the aggregator, as well as the notification mechanism, are the two most important components in this application.
+
+## 3. The Akka Receptionist: Publishing
+
+In order to find an actor under a name or identifier, it must be _registered_ by a unique identifier known as a `ServiceKey`. This is a simple data structure that is registered at the level of the actor system. After the registration is done, then the finder is able to query the actor system for all actors registered under that key, thereby retrieving their `ActorRef`s and sending them messages.
+
+This registration is done through a special actor known as the _receptionist_. All actor systems have a receptionist, and the goal of this actor is to perform this `ServiceKey`-`ActorRef`s mapping.
+
+For the main guardian actor to publish the fact that `dataAgg1` is the data aggregator to use, we need to define a `ServiceKey` by which we can identify the data aggregator. It's usually best practice to place the `ServiceKey` inside the object that spawns the actors to be registered. In this case, under the `DataAggregator` object:
+
+```scala
+// inside DataAggregator
+val serviceKey = ServiceKey[SensorReading]("dataAggregator")
+```
+
+The `ServiceKey` must be typed with the same type as the actor that we want to register. With the `ServiceKey` in place, we can then add some TODOs in our guardian actor:
+
+```scala
+val guardian: Behavior[NotUsed] = Behaviors.setup { context =>
+ // controller for the sensors
+ context.spawn(Sensor.controller(), "controller")
+
+ val dataAgg1 = context.spawn(DataAggregator(), "data_agg_1")
+ // "publish" dataAgg1 is available by associating it to a key (service key)
+ context.system.receptionist ! Receptionist.register(DataAggregator.serviceKey, dataAgg1)
+
+ // change data aggregator after 10s
+ Thread.sleep(10000)
+ context.log.info("[guardian] Changing data aggregator")
+ context.system.receptionist ! Receptionist.deregister(DataAggregator.serviceKey, dataAgg1)
+ val dataAgg2 = context.spawn(DataAggregator(), "data_agg_2")
+ context.system.receptionist ! Receptionist.register(DataAggregator.serviceKey, dataAgg2)
+
+ Behaviors.empty
+}
+```
+
+We send the `context.system.receptionist` some special messages to either register or deregister an actor. You can register multiple actors to the same `ServiceKey` if you want, but in this case we'll keep it to just one actor.
+
+The protocol handled by the receptionist is pretty rich, and you can be notified when the registration is complete by listening to the `Registered` message given back by the receptionist.
+
+## 3. The Akka Receptionist: Subscribing
+
+The other side is a bit more involved, because we can fetch information from the receptionist in multiple ways:
+
+- we can query the receptionist and listen back for one response
+- we can _subscribe_ for updates and receive a message _every time_ the association with a `ServiceKey` was changed
+
+We'll need to work on the `apply()` method of the `Sensor` object:
+
+```scala
+def apply(id: String): Behavior[SensorCommand] = Behaviors.setup { context =>
+ // subscribe to the receptionist using the service key
+ context.system.receptionist ! Receptionist.Subscribe(DataAggregator.serviceKey, ???)
+}
+```
+
+We would like to be automatically notified if there is any change in the association to the `ServiceKey`. The API allows us to pass the `ServiceKey` instance in question and an actor which can handle a `Listing` message that the receptionist will send with every update.
+
+However, because our Sensor actor is typed with `SensorCommand` and we also need to handle the listing message, we will need a message adapter. We discussed the message adapter technique in [another article](/akka-message-adapter/), so we will use it here. We will need to wrap the listing message into some other `SensorCommand` that we can handle later:
+
+```scala
+// new message
+case class ChangeDataAggregator(agg: Option[ActorRef[SensorReading]]) extends SensorCommand
+
+def apply(id: String): Behavior[SensorCommand] = Behaviors.setup { context =>
+ // use a message adapter to turn a receptionist listing into a SensorCommand
+ val receptionistSubscriber: ActorRef[Receptionist.Listing] = context.messageAdapter {
+ case DataAggregator.serviceKey.Listing(set) => ChangeDataAggregator(set.headOption)
+ }
+
+ // subscribe to the receptionist using the service key
+ context.system.receptionist ! Receptionist.Subscribe(DataAggregator.serviceKey, receptionistSubscriber)
+}
+```
+
+So in the `setup` method we have subscribed to the receptionist and are able to receive its listings, transformed into `ChangeDataAggregator` messages. We now need to handle them and keep track of the data aggregator that we have on hand at the moment:
+
+```scala
+def activeSensor(id: String, aggregator: Option[ActorRef[SensorReading]]): Behavior[SensorCommand] =
+ Behaviors.receiveMessage {
+ case SensorHeartbeat =>
+ // send the data to the aggregator
+ aggregator.foreach(_ ! SensorReading(id, Random.nextDouble() * 40))
+ Behaviors.same
+ case ChangeDataAggregator(newAgg) =>
+ // swap the aggregator for the new one
+ activeSensor(id, newAgg)
+ }
+```
+
+And with this message handler in place, the return value of the `apply()` method is going to be
+
+```scala
+ active(newReadings)
+```
+
+Currently, the code of the sensor actor looks like this:
+
+```scala
+object Sensor {
+ def apply(id: String): Behavior[SensorCommand] = Behaviors.setup { context =>
+ // use a message adapter to turn a receptionist listing into a SensorCommand
+ val receptionistSubscriber: ActorRef[Receptionist.Listing] = context.messageAdapter {
+ case DataAggregator.serviceKey.Listing(set) => ChangeDataAggregator(set.headOption)
+ }
+ // subscribe to the receptionist using the service key
+ context.system.receptionist ! Receptionist.Subscribe(DataAggregator.serviceKey, receptionistSubscriber)
+ activeSensor(id, None)
+ }
+
+ def activeSensor(id: String, aggregator: Option[ActorRef[SensorReading]]): Behavior[SensorCommand] =
+ Behaviors.receiveMessage {
+ case SensorHeartbeat =>
+ aggregator.foreach(_ ! SensorReading(id, Random.nextDouble() * 40))
+ Behaviors.same
+ case ChangeDataAggregator(newAgg) =>
+ activeSensor(id, newAgg)
+ }
+}
+```
+
+## 4. The Other Actors
+
+We need to finish off the `DataAggregator` and the sensor controller. The data aggregator will keep track of all the readings received so far from all the sensors, using a ["stateless"](/stateful-stateless-actors/) approach, which we also describe in another article:
+
+```scala
+ def apply(): Behavior[SensorReading] = active(Map())
+ def active(latestReadings: Map[String, Double]): Behavior[SensorReading] = Behaviors.receive { (context, reading) =>
+ val id = reading.id
+ val value = reading.value
+ // val SensorReading(id, value) = reading
+ val newReadings = latestReadings + (id -> value)
+ // "display" part - in real life this would feed a graph, a data ingestion engine or processor
+ context.log.info(s"[${context.self.path.name}] Latest readings: $newReadings")
+ active(newReadings)
+ }
+```
+
+and the sensor controller is a dumb actor which doesn't receive any messages:
+
+```scala
+ def controller(): Behavior[NotUsed] = Behaviors.setup { context =>
+ val sensors = (1 to 10).map(i => context.spawn(Sensor(s"sensor_$i"), s"sensor_$i"))
+ val logger = context.log // used so that we don't directly use context inside the lambda below
+ // send heartbeats every second
+ import context.executionContext
+ context.system.scheduler.scheduleAtFixedRate(1.second, 1.second) { () =>
+ logger.info("Heartbeat")
+ sensors.foreach(_ ! SensorHeartbeat)
+ }
+ Behaviors.empty
+ }
+```
+
+## 5. The Test
+
+If we run this application, we notice that every second, the sensor controller sends the heartbeat, and by some magic — we now know how it works — the sensors automatically know where to send their data, because the aggregator picks up the readings and displays all of them every second.
+
+After 10 seconds, the heartbeats keep running, but the logs now say `data_agg_2` — so the sensors were automatically notified that the data aggregator changed, so they simply pushed their readings elsewhere. Exactly as intended.
+
+The entire code looks like this:
+
+```scala
+case class SensorReading(id: String, value: Double)
+
+object DataAggregator {
+ val serviceKey = ServiceKey[SensorReading]("dataAggregator")
+ def apply(): Behavior[SensorReading] = active(Map())
+ def active(latestReadings: Map[String, Double]): Behavior[SensorReading] = Behaviors.receive { (context, reading) =>
+ val id = reading.id
+ val value = reading.value
+ // val SensorReading(id, value) = reading
+ val newReadings = latestReadings + (id -> value)
+ // "display" part
+ context.log.info(s"[${context.self.path.name}] Latest readings: $newReadings")
+ active(newReadings)
+ }
+}
+
+// sensor section
+trait SensorCommand
+case object SensorHeartbeat extends SensorCommand
+case class ChangeDataAggregator(agg: Option[ActorRef[SensorReading]]) extends SensorCommand
+
+object Sensor {
+ def apply(id: String): Behavior[SensorCommand] = Behaviors.setup { context =>
+ // use a message adapter to turn a receptionist listing into a SensorCommand
+ val receptionistSubscriber: ActorRef[Receptionist.Listing] = context.messageAdapter {
+ case DataAggregator.serviceKey.Listing(set) => ChangeDataAggregator(set.headOption)
+ }
+ // subscribe to the receptionist using the service key
+ context.system.receptionist ! Receptionist.Subscribe(DataAggregator.serviceKey, receptionistSubscriber)
+ activeSensor(id, None)
+ }
+ def activeSensor(id: String, aggregator: Option[ActorRef[SensorReading]]): Behavior[SensorCommand] =
+ Behaviors.receiveMessage {
+ case SensorHeartbeat =>
+ aggregator.foreach(_ ! SensorReading(id, Random.nextDouble() * 40))
+ Behaviors.same
+ case ChangeDataAggregator(newAgg) =>
+ activeSensor(id, newAgg)
+ }
+ def controller(): Behavior[NotUsed] = Behaviors.setup { context =>
+ val sensors = (1 to 10).map(i => context.spawn(Sensor(s"sensor_$i"), s"sensor_$i"))
+ val logger = context.log
+ // send heartbeats every second
+ import context.executionContext
+ context.system.scheduler.scheduleAtFixedRate(1.second, 1.second) { () =>
+ logger.info("Heartbeat")
+ sensors.foreach(_ ! SensorHeartbeat)
+ }
+ Behaviors.empty
+ }
+}
+
+val guardian: Behavior[NotUsed] = Behaviors.setup { context =>
+ // controller for the sensors
+ context.spawn(Sensor.controller(), "controller")
+ val dataAgg1 = context.spawn(DataAggregator(), "data_agg_1")
+ // "publish" dataAgg1 is available by associating it to a key (service key)
+ context.system.receptionist ! Receptionist.register(DataAggregator.serviceKey, dataAgg1)
+ // change data aggregator after 10s
+ Thread.sleep(10000)
+ context.log.info("[guardian] Changing data aggregator")
+ context.system.receptionist ! Receptionist.deregister(DataAggregator.serviceKey, dataAgg1)
+ val dataAgg2 = context.spawn(DataAggregator(), "data_agg_2")
+ context.system.receptionist ! Receptionist.register(DataAggregator.serviceKey, dataAgg2)
+ Behaviors.empty
+}
+
+def main(args: Array[String]): Unit = {
+ val system = ActorSystem(guardian, "ActorDiscovery")
+ import system.executionContext
+ system.scheduler.scheduleOnce(20.seconds, () => system.terminate())
+}
+```
+
+## 5. Conclusion
+
+In this article we discovered (pun) Akka Discovery, a powerful tool to find actors and use them in the situation where it's hard (or impossible) to locate the right reference for your needs. We saw where Discovery is useful, we learned how to use the actor system's receptionist to register, deregister and subscribe for updates, so that our actors can seamlessly send the right messages to the right actors.
diff --git a/_posts/2022-04-01-akka-cassandra-project.md b/_posts/2022-04-01-akka-cassandra-project.md
new file mode 100644
index 000000000000..150fa8f86961
--- /dev/null
+++ b/_posts/2022-04-01-akka-cassandra-project.md
@@ -0,0 +1,1203 @@
+---
+title: "A Scala project with Akka, Cats and Cassandra"
+date: 2022-04-01
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka, cassandra, cats]
+excerpt: "Akka, Cats and Cassandra in a bigger Scala project integrating multiple pieces in the Scala ecosystem."
+---
+
+_This mini-project is a collaboration between me (Daniel) and [Riccardo Cardin](https://github.com/rcardin), one of the prominent Rock the JVM alumni. Big thanks to Riccardo for contributing with most of the code that ended up in this article and on camera._
+
+This article will show you how to write a bigger project involving multiple libraries and tools in the Scala world. We're going to use:
+
+- Akka (typed) Actors for the business logic
+- Akka (typed) Persistence for event sourcing
+- Cassandra for storage
+- Akka HTTP for the REST API
+- Cats for data validation
+
+We're going to write a **mini-bank application**, where users can create bank accounts, retrieve their details and interact with their bank account to deposit/withdraw money.
+
+This article assumes that you're familiar with the basics of Akka Typed, Akka HTTP and Cats. We have various articles here on the blog (with corresponding videos) that will give you the essential tools, so feel free to click on the tags at the bottom, or search here on the blog, and you'll find everything you need. For in-depth mastery we have full-blown courses, so if you're interested, you can check out
+
+- the [Akka Typed course](https://rockthejvm.com/p/akka-essentials)
+- the [Akka HTTP course](https://rockthejvm.com/p/akka-http)
+- the [Cats course](https://rockthejvm.com/p/cats)
+
+We're going to write Scala 2 in this article for library compatibility reasons, although the exact same code will work on Scala 3 as well once the libraries have been updated. We're also going to need [Docker](https://docker.com) to start a Cassandra instance.
+
+The entire code is available [on GitHub](https://github.com/rockthejvm/akka-cassandra-demo).
+
+## Setup
+
+We'll start with a plain Scala SBT project. In your `build.sbt` file we'll add the necessary dependencies:
+
+```scala
+lazy val akkaHttpVersion = "10.2.8"
+lazy val akkaVersion = "2.6.9"
+lazy val circeVersion = "0.14.1"
+
+libraryDependencies ++= Seq(
+ "com.typesafe.akka" %% "akka-http" % akkaHttpVersion,
+ "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion,
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion,
+ "com.typesafe.akka" %% "akka-persistence-typed" % akkaVersion,
+ "com.datastax.oss" % "java-driver-core" % "4.13.0",
+ "com.typesafe.akka" %% "akka-persistence-cassandra" % "1.0.5",
+ "io.circe" %% "circe-core" % circeVersion,
+ "io.circe" %% "circe-generic" % circeVersion,
+ "io.circe" %% "circe-parser" % circeVersion,
+ "de.heikoseeberger" %% "akka-http-circe" % "1.39.2",
+ "ch.qos.logback" % "logback-classic" % "1.2.10",
+
+ // optional, if you want to add tests
+ "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % Test,
+ "com.typesafe.akka" %% "akka-actor-testkit-typed" % akkaVersion % Test,
+ "org.scalatest" %% "scalatest" % "3.2.9" % Test
+)
+```
+
+We'll also add a simple `docker-compose.yml` file to add a Cassandra service to our project:
+
+```yaml
+version: '3.8'
+
+services:
+ cassandra:
+ image: cassandra:4.0.3
+ ports:
+ - 9042:9042
+ environment:
+ - CASSANDRA_CLUSTER_NAME=akka-cassandra-cluster
+```
+
+After you add the docker-compose file, you can run `docker-compose up` in the directory where you created this file (ideally in the root dir of the project). This will download the necessary images and spin up the Docker container with the Cassandra service. While it downloads/spins up, you can go forward with this article.
+
+## The "Architecture"
+
+This is not a production application, but we do have some moving parts.
+
+We're going to create a mini-bank application which manages people's bank accounts. We need to support the following operations:
+
+- creating a bank account
+- retrieving current bank account details
+- depositing/withdrawing money
+
+Using Akka actors and Akka Persistence, the application will work as follows
+
+- each bank account is a persistent actor
+- all events are recorded (creation, update etc)
+- all events are replayed in case of failure/restart
+- one big bank (also persistent) actor manages all actors
+- a HTTP server with a REST API handles requests from outside
+- all events are stored in Cassandra
+
+The "architecture" therefore looks like this:
+
+
+
+We're going to split the work on this application in 4 parts:
+
+1. The bank account actor, using Akka Persistence
+2. The "main" bank actor, also using Akka Persistence
+3. The HTTP server with its REST API, using Akka HTTP
+4. The data validation, using Cats
+
+## 1. The Bank Accounts
+
+If you want to follow this chapter in video form, watch it below:
+
+{% include video id="PPIPGzrc2wo" provider="youtube" %}
+
+We're going to model bank accounts as independent, persistent actors. Each bank account will be its own actor, which will be created by the "main" bank actor either
+
+- upon user request
+- at application restart/in case of failure
+
+This bank account actor will also take care to treat all events strictly related to it, which are
+
+- creation
+- update
+- retrieval
+- (optionally) deletion, which we'll leave as an exercise
+
+So we'll start with a plain Scala object:
+
+```scala
+object PersistentBankAccount {
+ // our code here
+}
+```
+
+### 1.1. Data Modeling
+
+This bank account actor will need a few pieces of data:
+
+1. the messages it can receive — in Akka Persistence-speak they're called _commands_
+2. the events it will store in Cassandra
+3. the structures to manage its internal state
+4. the responses it may send to the main bank actor
+
+The commands will look as follows:
+
+```scala
+sealed trait Command
+object Command {
+ case class CreateBankAccount(user: String, currency: String, initialBalance: Double, replyTo: ActorRef[Response]) extends Command
+ case class UpdateBalance(id: String, currency: String, amount: Double /* can be < 0*/, replyTo: ActorRef[Response]) extends Command
+ case class GetBankAccount(id: String, replyTo: ActorRef[Response]) extends Command
+}
+```
+
+Obviously a simplification to how a production application would look like, but for the scope of this project should be complete enough. Notice all the commands have a `replyTo: ActorRef[Response]` in them, so that this actor knows who to send the response back to.
+
+A note: _please don't use the `Double` type to manage money_. The floating point standard cannot fully represent tenths and hundredths and can give (very) slightly incorrect results with multiplication and division in certain cases as a result. For our convenience in this project, we'll use `Double`.
+
+In terms of _events_, we need to store just enough so that we can reestablish the state of the bank account from these events if the application crashed for whatever reason. The data structures we'll use will look like this:
+
+```scala
+trait Event
+case class BankAccountCreated(bankAccount: BankAccount) extends Event
+case class BalanceUpdated(amount: Double) extends Event
+```
+
+As for state, we'll simply store the identifier of the account, the user's identifier, the currency and the current amount.
+
+```scala
+case class BankAccount(id: String, user: String, currency: String, balance: Double)
+```
+
+In terms of responses, we'll match them with the appropriate commands. They will look like this:
+
+```scala
+sealed trait Response
+object Response {
+ case class BankAccountCreatedResponse(id: String) extends Response
+ case class BankAccountBalanceUpdatedResponse(maybeBankAccount: Try[BankAccount]) extends Response
+ case class GetBankAccountResponse(maybeBankAccount: Option[BankAccount]) extends Response
+}
+```
+
+Notice that in the `BankAccountBalanceUpdatedResponse` class we use a `Try`, because the updating might fail for different reasons, for example:
+
+- the id of the bank account requested might be different from this bank account's id
+- the amount might be illegal, e.g. trying to withdraw more than you have
+
+An in the `GetBankAccountResponse` we use an `Option` because there are only two answers we're considering: either there is a bank account here, or it's not.
+
+### 1.2. The Bank Account Persistent Actor
+
+A persistent actor is defined in terms of four things:
+
+1. its unique persistence ID, which will be used to store data in the store and retrieve data from the store
+2. its state, which can change over time
+3. its message handler, aka _command_ handler
+4. its _event_ handler, i.e. what the actor does after storing an event to the persistent store, or _restoring_ an event after failure
+
+Let's take them in turn. First: the persistence ID — this one is straightforward, because the main bank actor will allocate a new UUID once it creates this actor in the first place, so we'll simply pass it on upon creation.
+
+Second: the state. We already know which data type we'll use (`BankAccount`), so we'll simply need to pass an "empty" state upon creation. Modifying the state happens when the actor receives a message (= a command) and when the actor handles an event, which are items 3 and 4 on the above list.
+
+Third: the command handler. This is a function that, given the current state and an incoming command, will produce an `Effect`. This `Effect` may involve sending messages, persisting items in the persistent store, changing state, or a combination of the above. The function signature is as follows:
+
+```scala
+val commandHandler: (BankAccount, Command) => Effect[Event, BankAccount] = (state, command) => // continue here
+```
+
+and we can also continue with the implementation — we can run a pattern match on the command and treat each possible message type in turn:
+
+```scala
+// continued
+command match {
+ case CreateBankAccount(user, currency, initialBalance, bank) =>
+ val id = state.id
+ Effect
+ .persist(BankAccountCreated(BankAccount(id, user, currency, initialBalance))) // persisted into Cassandra
+ .thenReply(bank)(_ => BankAccountCreatedResponse(id))
+ // continue here
+}
+```
+
+There's a lot of magic happening in these few lines. Breaking this down:
+
+- Before this account receives any messages, it needs to be created by the bank.
+- Upon creation of this account, the bank will send it a `CreateBankAccount` message.
+- Upon reception of the command from the main bank actor, the account will store a `BankAccountCreated` _event_ to Cassandra.
+- After storing the event, the _event handler_ will be invoked (the fourth item on the list, to be written shortly).
+- The account will then reply to the bank actor with a `BankAccountCreatedResponse`.
+- The bank will then surface the response to the HTTP layer, but that's none of our concern right now.
+
+Going forward with the other cases:
+
+```scala
+// continued
+case UpdateBalance(_, _, amount, bank) =>
+ val newBalance = state.balance + amount
+ // check here for withdrawal
+ if (newBalance < 0) // illegal
+ Effect.reply(bank)(BankAccountBalanceUpdatedResponse(Failure(new RuntimeException("Cannot withdraw more than available"))))
+ else
+ Effect
+ .persist(BalanceUpdated(amount))
+ .thenReply(bank)(newState => BankAccountBalanceUpdatedResponse(Success(newState)))
+case GetBankAccount(_, bank) =>
+ Effect.reply(bank)(GetBankAccountResponse(Some(state)))
+} // closing the pattern match
+```
+
+Following the structure of the first case,
+
+- if we attempt withdrawing more than we have available, we'll send back a response to the bank with a `Failure`
+- if the balance modification was successful, persist the appropriate event and send back a successful response
+- the "get" command simply responds with the current state of the account; can be improved with security checks, etc.
+
+That was the command handler, the third item on the list.
+
+Fourth: the event handler. This is a function which, given the current state of the actor and the event which has just been stored/restored, will return a new state of the actor. This assumes the event was successfully stored:
+
+```scala
+val eventHandler: (BankAccount, Event) => BankAccount = (state, event) =>
+ event match {
+ case BankAccountCreated(bankAccount) =>
+ bankAccount
+ case BalanceUpdated(amount) =>
+ state.copy(balance = state.balance + amount)
+ }
+```
+
+An important note: this same handler will be invoked both after the event is stored, and if the actor/application crashes and the actor is restarted: in such a case, the actor queries Cassandra for all events tied to its persistence ID and replays all events in sequence by invoking the `eventHandler` on each of them in turn, to recreate its latest state before crash/shutdown.
+
+The final bit ties everything together:
+```scala
+
+def apply(id: String): Behavior[Command] =
+ EventSourcedBehavior[Command, Event, BankAccount](
+ persistenceId = PersistenceId.ofUniqueId(id),
+ emptyState = BankAccount(id, "", "", 0.0), // unused
+ commandHandler = commandHandler,
+ eventHandler = eventHandler
+ )
+```
+
+The complete code will look like this:
+
+```scala
+package com.rockthejvm.bank.actors
+
+import akka.actor.typed.{ActorRef, Behavior}
+import akka.persistence.typed.PersistenceId
+import akka.persistence.typed.scaladsl.{Effect, EventSourcedBehavior}
+
+import scala.util.{Failure, Success, Try}
+
+// a single bank account
+object PersistentBankAccount {
+
+ // commands = messages
+ sealed trait Command
+ object Command {
+ case class CreateBankAccount(user: String, currency: String, initialBalance: Double, replyTo: ActorRef[Response]) extends Command
+ case class UpdateBalance(id: String, currency: String, amount: Double /* can be < 0*/, replyTo: ActorRef[Response]) extends Command
+ case class GetBankAccount(id: String, replyTo: ActorRef[Response]) extends Command
+ }
+
+ // events = to persist to Cassandra
+ trait Event
+ case class BankAccountCreated(bankAccount: BankAccount) extends Event
+ case class BalanceUpdated(amount: Double) extends Event
+
+ // state
+ case class BankAccount(id: String, user: String, currency: String, balance: Double)
+
+ // responses
+ sealed trait Response
+ object Response {
+ case class BankAccountCreatedResponse(id: String) extends Response
+ case class BankAccountBalanceUpdatedResponse(maybeBankAccount: Try[BankAccount]) extends Response
+ case class GetBankAccountResponse(maybeBankAccount: Option[BankAccount]) extends Response
+ }
+
+ import Command._
+ import Response._
+
+
+ val commandHandler: (BankAccount, Command) => Effect[Event, BankAccount] = (state, command) =>
+ command match {
+ case CreateBankAccount(user, currency, initialBalance, bank) =>
+ val id = state.id
+ Effect
+ .persist(BankAccountCreated(BankAccount(id, user, currency, initialBalance))) // persisted into Cassandra
+ .thenReply(bank)(_ => BankAccountCreatedResponse(id))
+ case UpdateBalance(_, _, amount, bank) =>
+ val newBalance = state.balance + amount
+ // check here for withdrawal
+ if (newBalance < 0) // illegal
+ Effect.reply(bank)(BankAccountBalanceUpdatedResponse(Failure(new RuntimeException("Cannot withdraw more than available"))))
+ else
+ Effect
+ .persist(BalanceUpdated(amount))
+ .thenReply(bank)(newState => BankAccountBalanceUpdatedResponse(Success(newState)))
+ case GetBankAccount(_, bank) =>
+ Effect.reply(bank)(GetBankAccountResponse(Some(state)))
+ }
+
+ val eventHandler: (BankAccount, Event) => BankAccount = (state, event) =>
+ event match {
+ case BankAccountCreated(bankAccount) =>
+ bankAccount
+ case BalanceUpdated(amount) =>
+ state.copy(balance = state.balance + amount)
+ }
+
+ def apply(id: String): Behavior[Command] =
+ EventSourcedBehavior[Command, Event, BankAccount](
+ persistenceId = PersistenceId.ofUniqueId(id),
+ emptyState = BankAccount(id, "", "", 0.0), // unused
+ commandHandler = commandHandler,
+ eventHandler = eventHandler
+ )
+}
+```
+
+## 2. The Main Bank Actor
+
+This section is also available on video:
+
+{% include video id="NpGYj_Zpwsk" provider="youtube" %}
+
+This actor will manage all the bank accounts, which means it will be a parent of those accounts. The bank actor will be the middle layer between the HTTP server and the actual persistent bank accounts, which will do the majority of the work.
+
+```scala
+object Bank {
+ // our code here
+}
+```
+
+The bank actor will also have to be persistent. Why? Because if the application crashes, no bank account can be magically revived. Only _after_ the accounts have been created and their persistence IDs assigned, will the bank accounts start replaying their events from Cassandra. Therefore, we'll also need to store some events here, so that if the application needs to start again, the bank actor will have the right information to start up the appropriate bank accounts.
+
+Therefore, we'll also need to manage
+
+1. commands
+2. events
+3. internal state
+4. responses
+
+Thankfully, the commands will be identical to the ones from the persistent bank accounts: creation, update, retrieval, (optionally) deletion. There's no need for us to change here. Same for the responses. Therefore, we'll import the commands and responses
+
+```scala
+import PersistentBankAccount.Command._
+import PersistentBankAccount.Response._
+import PersistentBankAccount.Command
+```
+
+and set up new data structures for events and state:
+
+```scala
+// events
+sealed trait Event
+case class BankAccountCreated(id: String) extends Event
+
+// state
+case class State(accounts: Map[String, ActorRef[Command]])
+```
+
+For events, we really only need to store the creation of the accounts so we know how to spawn the actors again. For state, we'll keep an internal map to retrieve actors by their unique identifier.
+
+Because the bank is also a persistent actor, we'll need
+
+1. a persistence ID
+2. an empty state
+3. a command handler
+4. an event handler
+
+The first two are straightforward: a `"bank"` and a `State` with an empty `Map()` should suffice.
+
+Third: the command handler. Again, a function taking the current state and an incoming command, and returning an `Effect`. The structure will look like this:
+
+```scala
+val commandHandler: (State, Command) => Effect[Event, State] = ???
+```
+
+However, we need to be able to spawn new bank accounts in this handler, which means we'll need an `ActorContext` to do that. It's usually available when we create the final behavior of the actor, so we need to be able to pass it here, so our definition will change to
+
+```scala
+def commandHandler(context: ActorContext[Command]): (State, Command) => Effect[Event, State] = (state, command) =>
+ command match {
+ // continue here
+ }
+```
+
+Already took the liberty of doing a pattern match on the command, which will treat all the cases, as follows:
+
+```scala
+case createCommand @ CreateBankAccount(_, _, _, _) =>
+ val id = UUID.randomUUID().toString
+ val newBankAccount = context.spawn(PersistentBankAccount(id), id)
+ Effect
+ .persist(BankAccountCreated(id))
+ .thenReply(newBankAccount)(_ => createCommand)
+```
+
+If we need to create an account, we'll generate a unique identifier, spawn a bank account actor, persist the creation event so we know how to bring the actor back if necessary, then (very importantly) pass the command down to the new actor.
+
+Further:
+
+```scala
+case updateCmd @ UpdateBalance(id, _, _, replyTo) =>
+ state.accounts.get(id) match {
+ case Some(account) =>
+ Effect.reply(account)(updateCmd)
+ case None =>
+ Effect.reply(replyTo)(BankAccountBalanceUpdatedResponse(Failure(new RuntimeException("Bank account cannot be found")))) // failed account search
+ }
+```
+
+If we need to add/withdraw money, we'll first need to find the account: if it's found, great — pass the command on to the account actor, if not, return a failure to whoever sent this command to the bank. Similarly for bank account retrieval:
+
+```scala
+case getCmd @ GetBankAccount(id, replyTo) =>
+ state.accounts.get(id) match {
+ case Some(account) =>
+ Effect.reply(account)(getCmd)
+ case None =>
+ Effect.reply(replyTo)(GetBankAccountResponse(None)) // failed search
+ }
+```
+
+That was the third item on the list.
+
+Fourth: the event handler. In this case, we only have one event to handle, which is the bank account creation. There is only one problem:
+
+- If the bank actor is in "active" mode, i.e. normally receiving commands, the event handler occurs after persisting the `BankAccountCreated` event, and therefore the bank account actor exists.
+- If the bank actor is in recovery mode, i.e. on application start/restart, the event handler occurs after retrieving the `BankAccountCreated` event from Cassandra, and therefore the bank account actor _needs to be created here_.
+
+These points considered, we have
+
+```scala
+def eventHandler(context: ActorContext[Command]): (State, Event) => State = (state, event) =>
+ event match {
+ case BankAccountCreated(id) =>
+ val account = context.child(id) // exists after command handler,
+ .getOrElse(context.spawn(PersistentBankAccount(id), id)) // does NOT exist in the recovery mode, so needs to be created
+ .asInstanceOf[ActorRef[Command]] // harmless, it already has the right type
+ state.copy(state.accounts + (id -> account))
+ }
+```
+
+Then, all we need to do is to write an `apply` method which will return the appropriate bank behavior:
+
+```scala
+// behavior
+def apply(): Behavior[Command] = Behaviors.setup { context =>
+ EventSourcedBehavior[Command, Event, State](
+ persistenceId = PersistenceId.ofUniqueId("bank"),
+ emptyState = State(Map()),
+ commandHandler = commandHandler(context),
+ eventHandler = eventHandler(context)
+ )
+}
+```
+
+Amazing. Let's try this.
+
+### 2.1. Testing the Actors
+
+For interaction with Cassandra, we're going to need a configuration to use the Cassandra journal, so in `src/main/resources`, we'll add an `application.conf` file with the following configuration:
+
+```properties
+# Journal
+akka.persistence.journal.plugin = "akka.persistence.cassandra.journal"
+akka.persistence.cassandra.journal.keyspace-autocreate = true
+akka.persistence.cassandra.journal.tables-autocreate = true
+datastax-java-driver.advanced.reconnect-on-init = true
+
+# Snapshot
+akka.persistence.snapshot-store.plugin = "akka.persistence.cassandra.snapshot"
+akka.persistence.cassandra.snapshot.keyspace-autocreate = true
+akka.persistence.cassandra.snapshot.tables-autocreate = true
+
+akka.actor.allow-java-serialization = on
+```
+
+We can of course use the Akka TestKit to test the actors, but we'll go for some live testing with some events stored in Cassandra! With the Docker container up and running — all you need is to run `docker-compose up` in the root directory — we'll write some quick application to run with the Bank actor and some creation/retrieval commands:
+
+```scala
+object BankPlayground {
+ import PersistentBankAccount.Command._
+ import PersistentBankAccount.Response._
+ import PersistentBankAccount.Response
+
+ def main(args: Array[String]): Unit = {
+ val rootBehavior: Behavior[NotUsed] = Behaviors.setup { context =>
+ val bank = context.spawn(Bank(), "bank")
+ val logger = context.log
+
+ val responseHandler = context.spawn(Behaviors.receiveMessage[Response]{
+ case BankAccountCreatedResponse(id) =>
+ logger.info(s"successfully created bank account $id")
+ Behaviors.same
+ case GetBankAccountResponse(maybeBankAccount) =>
+ logger.info(s"Account details: $maybeBankAccount")
+ Behaviors.same
+ }, "replyHandler")
+
+ // ask pattern
+ import akka.actor.typed.scaladsl.AskPattern._
+ import scala.concurrent.duration._
+ implicit val timeout: Timeout = Timeout(2.seconds)
+ implicit val scheduler: Scheduler = context.system.scheduler
+ implicit val ec: ExecutionContext = context.executionContext
+
+ // test 1
+ // bank ! CreateBankAccount("daniel", "USD", 10, responseHandler)
+
+ // test 2
+ // bank ! GetBankAccount("replaceWithYourUuidHere", responseHandler)
+
+ Behaviors.empty
+ }
+
+ val system = ActorSystem(rootBehavior, "BankDemo")
+ }
+}
+```
+
+For this live test, we're first going to send a creation message and check that there are two events in Cassandra (one from the bank and one from the account). Running this application should give us the log `successfully created bank account ...`. You can then shut down the application and run it again, this time with just the second message. A successful log with the account details proves multiple things:
+
+- that the bank actor works
+- that the account actor works
+- that the bank account can successfully respawn the account
+- that the account can successfully restore its state
+
+Sure enough, we can also inspect Cassandra for the relevant events. While Cassandra is running, from another terminal we can run
+
+```bash
+docker ps
+```
+
+and look at the container name, copy it, and then run
+
+```bash
+docker exec -it akka-cassandra-demo_cassandra_1 cqlsh
+```
+
+which will open the CQL prompt for us to inspect the tables. Inside, we'll run
+
+```sql
+select * from akka.messages;
+```
+
+and lo and behold, we have messages there! The Cassandra tables were created automatically by the Akka Persistence Cassandra journal.
+
+```text
+ persistence_id | partition_nr | sequence_nr | timestamp | event | event_manifest | meta | meta_ser_id | meta_ser_manifest | ser_id | ser_manifest | tags | timebucket | writer_uuid
+--------------------------------------+--------------+-------------+--------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+------+-------------+-------------------+--------+--------------+------+---------------+--------------------------------------
+ bank | 0 | 1 | 34633920-b1ce-11ec-8405-a1782b503565 | 0xaced000573720032636f6d2e726f636b7468656a766d2e62616e6b2e6163746f72732e42616e6b2442616e6b4163636f756e74437265617465646aebebd03a53f5500200014c000269647400124c6a6176612f6c616e672f537472696e673b787074002434373166373833392d633363392d346532352d393035622d326333626533343132383437 | | null | null | null | 1 | | null | 1648825200000 | 0dd3a479-9563-488b-8b72-d259bbaf5f8f
+ 471f7839-c3c9-4e25-905b-2c3be3412847 | 0 | 1 | 346c87f0-b1ce-11ec-8405-a1782b503565 | 0xaced000573720043636f6d2e726f636b7468656a766d2e62616e6b2e6163746f72732e50657273697374656e7442616e6b4163636f756e742442616e6b4163636f756e744372656174656493750afb52eb6b5b0200014c000b62616e6b4163636f756e7474003e4c636f6d2f726f636b7468656a766d2f62616e6b2f6163746f72732f50657273697374656e7442616e6b4163636f756e742442616e6b4163636f756e743b78707372003c636f6d2e726f636b7468656a766d2e62616e6b2e6163746f72732e50657273697374656e7442616e6b4163636f756e742442616e6b4163636f756e74d653249c8fb35b6d02000444000762616c616e63654c000863757272656e63797400124c6a6176612f6c616e672f537472696e673b4c0002696471007e00044c00047573657271007e00047870402400000000000074000355534474002434373166373833392d633363392d346532352d393035622d32633362653334313238343774000664616e69656c | | null | null | null | 1 | | null | 1648825200000 | 4f9cea0e-13d3-403f-8574-15f19a3a5664
+```
+
+Let's move on to the HTTP server.
+
+## 3. The HTTP Server
+
+This section is also available on video:
+
+{% include video id="q9psVBnMqJk" provider="youtube" %}
+
+For this section, we're going to use Akka HTTP (obviously), and we'll use the high-level DSL. We will expose the following endpoints:
+
+- A POST endpoint on `/bank` with a JSON payload that will create a new bank account; this will return the status of the request and the unique ID of the account in an HTTP header.
+- A GET endpoint on `/bank/(an id)` which returns a JSON payload containing the details of the account identified by that ID.
+- A PUT endpoint on `/bank/(an id)` with a JSON payload that will signify a withdrawal/deposit to the account; this will return a new JSON containing the new details of the account.
+
+Under a new `BankRouter` file, we need to represent the JSON payloads of these requests. We only have two:
+
+```scala
+case class BankAccountCreationRequest(user: String, currency: String, balance: Double)
+case class BankAccountUpdateRequest(currency: String, amount: Double)
+```
+
+As for responses, we already have the right data structures in the bank account definition, so we can either use them verbatim, or create new case classes with the same structure and some conversion methods to/from the responses from the bank account actor. For this article, we'll choose the former, so we'll simply
+
+```scala
+import com.rockthejvm.bank.actors.PersistentBankAccount.Response
+import com.rockthejvm.bank.actors.PersistentBankAccount.Response._
+```
+
+Plus a failure response in case any request is not right:
+
+```scala
+case class FailureResponse(reason: String)
+```
+
+Awesome. Now we need to be able to automatically serialize these case classes to and from JSON, so we'll add the Circe support with the imports
+
+```scala
+import io.circe.generic.auto._
+import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
+```
+
+And along with the main import of all directives in Akka HTTP
+
+```scala
+import akka.http.scaladsl.server.Directives._
+```
+
+we can then get started with the Akka HTTP routes we'll need for the server.
+
+### 3.1. Creating a Bank Account in the Akka HTTP Server
+
+Starting with the first endpoint, a POST on `/bank`:
+
+```scala
+object BankRouter {
+ val routes =
+ pathPrefix("bank") {
+ pathEndOrSingleSlash {
+ post {
+ // parse the payload
+ entity(as[BankAccountCreationRequest]) { request =>
+ /*
+ TODO 1
+ */
+ }
+ }
+ }
+ }
+}
+```
+
+Inside, we need to do the following:
+
+1. fetch the bank actor
+2. send it a `CreateBankAccount` _command_ — note that it's different from the HTTP request
+3. parse its reply
+4. send back an HTTP response
+
+First, we need the bank actor, which we don't have. We can receive it as a constructor argument to this `BankRouter`, which means we'll need to make it a class. Besides, we'll also need an `ActorSystem` to be able to run the directives, so we'll pass this one too, as an `implicit` argument, or a `using` clause in Scala 3.
+
+```scala
+// at the top
+import akka.actor.typed.{ActorRef, ActorSystem}
+import com.rockthejvm.bank.actors.PersistentBankAccount.Command
+import com.rockthejvm.bank.actors.PersistentBankAccount.Command._
+
+// change here
+class BankRouter(bank: ActorRef[Command])(implicit system: ActorSystem[_]) {
+ // same routes
+}
+```
+
+For point number two, we need to convert the HTTP request into a command we can pass to the bank actor:
+
+```scala
+case class BankAccountCreationRequest(user: String, currency: String, balance: Double) {
+ // added now
+ def toCommand(replyTo: ActorRef[Response]): Command = CreateBankAccount(user, currency, balance, replyTo)
+}
+```
+
+For point number three, we can use the bank actor to send a command and expect a reply. We'll use the ask pattern to do this.
+
+```scala
+// at the top
+import akka.actor.typed.scaladsl.AskPattern._
+import akka.util.Timeout
+import scala.concurrent.duration._
+
+// within BankRouter
+implicit val timeout: Timeout = Timeout(5.seconds)
+def createBankAccount(request: BankAccountCreationRequest): Future[Response] =
+ bank.ask(replyTo => request.toCommand(replyTo))
+```
+
+The ask pattern is useful for this kind of one-off, request-response interaction. Akka will create an intermediate actor with a short lifespan, which will serve as the destination for the eventual response, and this actor will fulfill a `Future` with that response when it receives it. It is this `Future` that we can then handle in our "regular", non-actor code.
+
+Finally, point four is our `TODO 1`:
+
+```scala
+// instead of TODO 1
+onSuccess(createBankAccount(request)) {
+ // send back an HTTP response
+ case BankAccountCreatedResponse(id) =>
+ respondWithHeader(Location(s"/bank/$id")) {
+ complete(StatusCodes.Created)
+ }
+}
+```
+
+`onSuccess` is a directive that asynchronously waits for a Future to be completed, and once it's done, the content of the Future is subject to the function below, which needs to return another directive: in our case, we'll return an HTTP 201, and will return the URI of the bank account as a `Location` HTTP header.
+
+### 3.2. Retrieving a Bank Account
+
+Our routes currently look like this:
+
+```scala
+ val routes =
+ pathPrefix("bank") {
+ pathEndOrSingleSlash {
+ post {
+ // parse the payload
+ entity(as[BankAccountCreationRequest]) { request =>
+ onSuccess(createBankAccount(request)) {
+ // send back an HTTP response
+ case BankAccountCreatedResponse(id) =>
+ respondWithHeader(Location(s"/bank/$id")) {
+ complete(StatusCodes.Created)
+ }
+ }
+ }
+ }
+ }
+ }
+```
+
+The second endpoint is a GET on `/bank/someUUID`, so we need to add another route inside `pathPrefix("bank")`:
+
+```scala
+ val routes =
+ pathPrefix("bank") {
+ pathEndOrSingleSlash {
+ // same code
+ } ~ // <-- careful with this one
+ path(Segment) { id =>
+ get {
+ // TODO 2
+ }
+ }
+ }
+```
+
+We need to make Akka HTTP parse the next token after `/bank` and return that token to us as the identifier of the account. Once again, we need to
+
+1. send a command to the bank actor to retrieve the details
+2. parse the response
+3. send back an HTTP response
+
+We'll follow the same pattern as before, so we'll add a method to ask the bank actor for some account details:
+
+```scala
+ def getBankAccount(id: String): Future[Response] =
+ bank.ask(replyTo => GetBankAccount(id, replyTo))
+```
+
+And we'll parse the response and send back a proper HTTP response in `TODO 2`:
+
+```scala
+onSuccess(getBankAccount(id)) {
+ case GetBankAccountResponse(Some(account)) =>
+ complete(account)
+ case GetBankAccountResponse(None) =>
+ complete(StatusCodes.NotFound, FailureResponse(s"Bank account $id cannot be found."))
+}
+```
+
+We complete the response with the account details passed directly as the case class instance, because the implicit marshallers will take care to serialize that instance to JSON.
+
+### 3.3. Updating a Bank Account
+
+Our routes now look like this:
+
+```scala
+ val routes =
+ pathPrefix("bank") {
+ pathEndOrSingleSlash {
+ post {
+ // parse the payload
+ entity(as[BankAccountCreationRequest]) { request =>
+ onSuccess(createBankAccount(request)) {
+ // send back an HTTP response
+ case BankAccountCreatedResponse(id) =>
+ respondWithHeader(Location(s"/bank/$id")) {
+ complete(StatusCodes.Created)
+ }
+ }
+ }
+ }
+ } ~
+ path(Segment) { id =>
+ get {
+ onSuccess(getBankAccount(id)) {
+ case GetBankAccountResponse(Some(account)) =>
+ complete(account)
+ case GetBankAccountResponse(None) =>
+ complete(StatusCodes.NotFound, FailureResponse(s"Bank account $id cannot be found."))
+ }
+ }
+ }
+ }
+```
+
+We need to add a third endpoint, which is a PUT on the same `/bank/UUID` path, therefore:
+
+```scala
+ val routes =
+ pathPrefix("bank") {
+ pathEndOrSingleSlash {
+ // endpoint 1
+ } ~
+ path(Segment) { id =>
+ get {
+ // endpoint 2
+ } ~ // <-- watch this one
+ put {
+ entity(as[BankAccountUpdateRequest]) { request => // need to parse the request
+ // TODO 3
+ }
+ }
+ }
+ }
+```
+
+Following the same pattern, in this endpoint we need to both parse the HTTP request's payload _and_ send back an HTTP response with a payload. So, same deal:
+
+1. Ask the bank actor to update the bank account.
+2. Expect a reply.
+3. Send back an HTTP response.
+
+Adding a method to ask the bank actor for the update:
+
+```scala
+def updateBankAccount(id: String, request: BankAccountUpdateRequest): Future[Response] =
+ bank.ask(replyTo => request.toCommand(id, replyTo))
+```
+
+After that, we need to invoke this method in an `onSuccess` directive like last time, and return an HTTP response in kind:
+
+```scala
+// instead of TODO 3
+onSuccess(updateBankAccount(id, request)) {
+ case BankAccountBalanceUpdatedResponse(Success(account)) =>
+ complete(account)
+ case BankAccountBalanceUpdatedResponse(Failure(ex)) =>
+ complete(StatusCodes.BadRequest, FailureResponse(s"${ex.getMessage}"))
+}
+```
+
+Our routes, therefore, will turn into this:
+
+```scala
+ val routes =
+ pathPrefix("bank") {
+ pathEndOrSingleSlash {
+ post {
+ // parse the payload
+ entity(as[BankAccountCreationRequest]) { request =>
+ onSuccess(createBankAccount(request)) {
+ // send back an HTTP response
+ case BankAccountCreatedResponse(id) =>
+ respondWithHeader(Location(s"/bank/$id")) {
+ complete(StatusCodes.Created)
+ }
+ }
+ }
+ }
+ } ~
+ path(Segment) { id =>
+ get {
+ onSuccess(getBankAccount(id)) {
+ case GetBankAccountResponse(Some(account)) =>
+ complete(account)
+ case GetBankAccountResponse(None) =>
+ complete(StatusCodes.NotFound, FailureResponse(s"Bank account $id cannot be found."))
+ }
+ } ~
+ put {
+ entity(as[BankAccountUpdateRequest]) { request =>
+ onSuccess(updateBankAccount(id, request)) {
+ case BankAccountBalanceUpdatedResponse(Success(account)) =>
+ complete(account)
+ case BankAccountBalanceUpdatedResponse(Failure(ex)) =>
+ complete(StatusCodes.BadRequest, FailureResponse(s"${ex.getMessage}"))
+ }
+ }
+ }
+ }
+ }
+```
+
+Please watch carefully where the `~` operator is present: this is an operator to chain routes. Every HTTP request is matched by the routes of the server in turn; if the HTTP request was not matched by the current route, it will try the next one through the `~` operator.
+
+Testing time!
+
+### 3.4. Testing the HTTP Server
+
+Much like we did earlier with testing the entire actor interaction, we'll also create a standalone application that will spin up an `ActorSystem`, create the bank actor, and start a new HTTP server based on it.
+
+For the `ActorSystem`, we need to be able to retrieve the `Bank` actor from inside of it, so we'll need to send it a message and expect a response:
+
+```scala
+import akka.actor.typed.{ActorRef, ActorSystem, Behavior}
+import akka.actor.typed.scaladsl.Behaviors
+import com.rockthejvm.bank.actors.Bank
+import com.rockthejvm.bank.actors.PersistentBankAccount.Command
+import akka.actor.typed.scaladsl.AskPattern._
+import akka.http.scaladsl.Http
+import akka.util.Timeout
+import com.rockthejvm.bank.http.BankRouter
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+import scala.util.{Success, Failure}
+
+object BankApp {
+ trait RootCommand
+ case class RetrieveBankActor(replyTo: ActorRef[ActorRef[Command]]) extends RootCommand
+
+ val rootBehavior: Behavior[RootCommand] = Behaviors.setup { context =>
+ val bankActor = context.spawn(Bank(), "bank")
+ Behaviors.receiveMessage {
+ case RetrieveBankActor(replyTo) =>
+ replyTo ! bankActor
+ Behaviors.same
+ }
+ }
+
+ // continue here
+}
+```
+
+Starting the HTTP server based on the bank actor will need some dedicated code as well:
+
+```scala
+def startHttpServer(bank: ActorRef[Command])(implicit system: ActorSystem[_]): Unit = {
+ implicit val ec: ExecutionContext = system.executionContext
+ val router = new BankRouter(bank)
+ val routes = router.routes
+
+ // start the server
+ val httpBindingFuture = Http().newServerAt("localhost", 8080).bind(routes)
+
+ // manage the server binding
+ httpBindingFuture.onComplete {
+ case Success(binding) =>
+ val address = binding.localAddress
+ system.log.info(s"Server online at http://${address.getHostString}:${address.getPort}")
+ case Failure(ex) =>
+ system.log.error(s"Failed to bind HTTP server, because: $ex")
+ system.terminate()
+ }
+}
+```
+
+And in the main method, we now need to bring all pieces together:
+
+```scala
+ def main(args: Array[String]): Unit = {
+ implicit val system: ActorSystem[RootCommand] = ActorSystem(rootBehavior, "BankSystem")
+ implicit val timeout: Timeout = Timeout(5.seconds)
+ implicit val ec: ExecutionContext = system.executionContext
+
+ // using the ask pattern again
+ val bankActorFuture: Future[ActorRef[Command]] = system.ask(replyTo => RetrieveBankActor(replyTo))
+ bankActorFuture.foreach(startHttpServer)
+ }
+```
+
+Run the application and have fun with the endpoints now, our application should be complete!
+
+## 4. Data Validation
+
+This bit is optional, because the application should work at this point. However, it's worth making our mini-bank a bit more robust in the face of malformed requests, and we can do this with the Cats validation capabilities. We talk about data validation and the Validated type in detail in the [Cats course](https://rockthejvm.com/p/cats), but here we're not going to need too much.
+
+We'll make a simple object where we'll store a generic mini-library for data validation.
+
+```scala
+import cats.data.ValidatedNel
+import cats.implicits._
+
+object Validation {
+ // based on cats.Validated
+ type ValidationResult[A] = ValidatedNel[ValidationFailure, A]
+
+ // validation failures
+ trait ValidationFailure {
+ def errorMessage: String
+ }
+
+ // continue here
+}
+```
+
+In a `ValidatedNel` (Nel = non-empty list), we always have either
+- a value of type `A` (the desired value)
+- a non-empty list of `ValidationFailure`s
+
+This data type is extremely useful, because we can accumulate _multiple_ errors with an HTTP request and surface them out to the user, instead of a single generic error message.
+
+Let's say, for instance, that we would often need to validate whether a field is present in an HTTP request, or that a numerical field satisfies some minimal properties (e.g. a bank account balance will not be negative).
+
+```scala
+// field must be present
+trait Required[A] extends (A => Boolean)
+// minimum value
+trait Minimum[A] extends ((A, Double) => Boolean) // for numerical fields
+trait MinimumAbs[A] extends ((A, Double) => Boolean) // for numerical fields
+```
+
+Let's further assume that for certain types, e.g. `Int` or `String`, we already have some instances that make sense all (or almost all) the time:
+
+```scala
+// would be `given` instances in Scala 3
+implicit val requiredString: Required[String] = _.nonEmpty
+implicit val minimumInt: Minimum[Int] = _ >= _
+implicit val minimumDouble: Minimum[Double] = _ >= _
+implicit val minimumIntAbs: MinimumAbs[Int] = Math.abs(_) >= _
+implicit val minimumDoubleAbs: MinimumAbs[Double] = Math.abs(_) >= _
+```
+
+An "internal" validation API that would use these instances would look something like this:
+
+```scala
+case class EmptyField(fieldName: String) extends ValidationFailure {
+ override def errorMessage = s"$fieldName is empty"
+}
+
+case class NegativeValue(fieldName: String) extends ValidationFailure {
+ override def errorMessage = s"$fieldName is negative"
+}
+
+case class BelowMinimumValue(fieldName: String, min: Double) extends ValidationFailure {
+ override def errorMessage = s"$fieldName is below the minimum threshold $min"
+}
+```
+
+Now, in terms of something that we would offer to the outside world in terms of data validation, we can expose general APIs for every type of validation we need, in our case "required field", "above a minimum value", "above a minimum value in absolute value".
+
+```scala
+// "main" API
+def validateMinimum[A: Minimum](value: A, threshold: Double, fieldName: String): ValidationResult[A] = {
+ if (minimum(value, threshold)) value.validNel
+ else if (threshold == 0) NegativeValue(fieldName).invalidNel
+ else BelowMinimumValue(fieldName, threshold).invalidNel
+}
+
+def validateMinimumAbs[A: MinimumAbs](value: A, threshold: Double, fieldName: String): ValidationResult[A] = {
+ if (minimumAbs(value, threshold)) value.validNel
+ else BelowMinimumValue(fieldName, threshold).invalidNel
+}
+
+def validateRequired[A: Required](value: A, fieldName: String): ValidationResult[A] =
+ if (required(value)) value.validNel
+ else EmptyField(fieldName).invalidNel
+```
+
+The `validNel` and `invalidNel` are extension methods allowed by the `cats.implicits._` import, so that we can build our `ValidationResult`s more easily.
+
+A general type class we can also expose is some sort of validator for any type, not just for those that pass certain predicates:
+
+```scala
+trait Validator[A] {
+ def validate(value: A): ValidationResult[A]
+}
+
+def validateEntity[A](value: A)(implicit validator: Validator[A]): ValidationResult[A] =
+ validator.validate(value)
+```
+
+We will use this type class for our HTTP requests that we need to validate, namely
+- the bank account creation request
+- the bank account update request
+
+We show in the [Advanced Scala course](https://rockthejvm.com/p/advanced-scala) that when we have a single implicit behavior that makes sense for a type, we should place that implicit value in the companion of that type. In our case, we'll place the implicit type class instances in the companion objects of these requests:
+
+```scala
+import cats.implicits._
+
+object BankAccountCreationRequest {
+ implicit val validator: Validator[BankAccountCreationRequest] = new Validator[BankAccountCreationRequest] {
+ override def validate(request: BankAccountCreationRequest): ValidationResult[BankAccountCreationRequest] = {
+ val userValidation = validateRequired(request.user, "user")
+ val currencyValidation = validateRequired(request.currency, "currency")
+ val balanceValidation = validateMinimum(request.balance, 0, "balance")
+ .combine(validateMinimumAbs(request.balance, 0.01, "balance"))
+
+ (userValidation, currencyValidation, balanceValidation).mapN(BankAccountCreationRequest.apply)
+ }
+ }
+}
+```
+
+We validate each field with the predicate that we need. Notice the use of `combine` which can aggregate multiple errors with a value, if that value invalidates both conditions. Also notice the handy use of `mapN`, which can aggregate _all_ the errors from the `Validated` instances in one convenient call. This is possible because `Validated` is an _applicative_, something we prove and deconstruct in the Cats course.
+
+We can follow the same pattern with the bank account update request:
+
+```scala
+object BankAccountUpdateRequest {
+ implicit val validator: Validator[BankAccountUpdateRequest] = new Validator[BankAccountUpdateRequest] {
+ override def validate(request: BankAccountUpdateRequest): ValidationResult[BankAccountUpdateRequest] = {
+ val currencyValidation = validateRequired(request.currency, "currency")
+ val amountValidation = validateMinimumAbs(request.amount, 0.01, "amount")
+
+ (currencyValidation, amountValidation).mapN(BankAccountUpdateRequest.apply)
+ }
+ }
+}
+```
+
+And with that, we have some implicit type class instances ready for the HTTP requests that we want to check. Now, we need to introduce this validation logic in the HTTP server. Inside the `BankRouter`, we'll add a method that will
+
+- try to validate a request based on an implicit `Validator` for that type
+- if the result is valid, follow the happy path, i.e. the normal route
+- if the result is invalid, return an HTTP response with a `FailureResponse` aggregating _all_ the errors tha were discovered
+
+The method will look like this:
+
+```scala
+def validateRequest[R: Validator](request: R)(routeIfValid: Route): Route =
+ validateEntity(request) match {
+ case Valid(_) =>
+ routeIfValid
+ case Invalid(failures) =>
+ complete(StatusCodes.BadRequest, FailureResponse(failures.toList.map(_.errorMessage).mkString(", ")))
+ }
+```
+
+We specifically made this method curried, because we would like to wrap our existing routes with a method call, in the style of
+
+```scala
+validateRequest(req) {
+ // allRoutesBelow
+}
+```
+
+And this is exactly what we'll do. Right after the `entity(as[...])` calls (of which we have two), we'll insert our `validateRequest` calls:
+
+```scala
+// in endpoint 1
+entity(as[BankAccountCreationRequest]) { request =>
+ validateRequest(request) { // <-- added here
+ onSuccess(createBankAccount(request)) {
+ // send back an HTTP response
+ case BankAccountCreatedResponse(id) =>
+ respondWithHeader(Location(s"/bank/$id")) {
+ complete(StatusCodes.Created)
+ }
+ }
+ }
+}
+```
+
+```scala
+// in endpoint 3
+entity(as[BankAccountUpdateRequest]) { request =>
+ validateRequest(request) { // <-- added here
+ onSuccess(updateBankAccount(id, request)) {
+ // send HTTP response
+ case BankAccountBalanceUpdatedResponse(Success(account)) =>
+ complete(account)
+ case BankAccountBalanceUpdatedResponse(Failure(ex)) =>
+ complete(StatusCodes.BadRequest, FailureResponse(s"${ex.getMessage}"))
+ }
+ }
+}
+```
+
+And this will conclude our data validation attempt — it doesn't _add_ new endpoints or functionality, but, as we all know as developers, surfacing descriptive error messages makes all the difference in the world. This approach was extremely simplified and there are many ways we can improve it and make it more robust while decoupled from the logic of the HTTP server itself.
+
+At this point, you can run the application again, and have fun with the new endpoints which give you much richer information, especially if you passed the wrong kind of data inside.
+
+## 5. Conclusion
+
+This was a whirlwind tutorial on how to use Akka Actors, Akka Persistence, Akka HTTP, Cassandra and Cats into a bigger application. We created persistent actors, we managed them with a "root" actor, we ran an HTTP server with a sleek DSL and REST API, and we made our data validation more robust with a bit of Cats and the Validated type. We hope you had as much fun writing this application as we did.
diff --git a/_posts/2022-05-12-scala-generics.md b/_posts/2022-05-12-scala-generics.md
new file mode 100644
index 000000000000..3d1f8b14088e
--- /dev/null
+++ b/_posts/2022-05-12-scala-generics.md
@@ -0,0 +1,226 @@
+---
+title: "Scala Generics: A Gentle Introduction"
+date: 2022-05-12
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, beginners]
+excerpt: "Scala generics are easy to understand if you come from a Java background. But what about Python or JS folks?"
+---
+
+This beginner-friendly article will introduce you to Scala generics in a smooth, approachable way. If you're just getting started with Scala and generics seem challenging to you, we'll make everything clear. The article is aimed at people getting into Scala from other languages _with dynamic typing_, such as Python or JavaScript. In my experience teaching, I noticed that Scala's type system, particularly generics, [variance](/scala-variance-positions/) and other bits around generics seem to be especially difficult if my students' entire programming experience was in Python, JS or other languages with dynamic type systems. That said, Java folks will also benefit from this article.
+
+This is a Scala generics guide that will explain
+- what generics are in Scala
+- where and how to use them
+- most importantly, _why_ they exist
+
+The concept of Scala generics is explained in-depth and with practice exercises in the [Scala Essentials course](https://rockthejvm.com/p/scala). The code that we write in this article works on both Scala 2 and Scala 3.
+
+If you want to see this article in video form, please watch below:
+
+{% include video id="ozcY_K-ij20" provider="youtube" %}
+
+## 1. Introduction to Scala Generics
+
+
+
+If you've started to learn Scala, you probably know that Scala has a _static type system_. By this we mean that the types of all the values, variables and expressions in your program are known by the compiler ahead of time, before the program runs. The compiler is therefore a tool to help us catch bugs in case we call some method or operator that is not allowed for those particular types. That's not the case for languages like Python or JavaScript; and there are lots of reasons to adopt a static or dynamic type system, and we won't need to go to deep there.
+
+In Scala, when we create a list, we write
+
+```scala
+val aList = List(1,2,3)
+```
+
+whereas in Python or JavaScript we write
+
+```python
+aList = [1,2,3]
+```
+
+Not much different, but in the case of the Scala compiler, the compiler already knows that `aList` is a list of integers, therefore you know for sure that when you write
+
+```scala
+val aNumber = aList(2)
+```
+
+that is an integer, whereas in the other cases (Python, JS, etc) the list might contain ints, strings and other data structures in the same place!
+
+We say that the type of `aList` is `List[Int]`, a list of integers. The `[Int]` is a _type argument_, which gives us a clear indication that all numbers in this list are integers: we can add integers, extract integers, transform integers, etc. Alternatively, we say that the type `List` is _generic_, meaning that you can use the list concept on many different types, and work with lists of ints, with lists of strings and everything else.
+
+For now, remember this:
+
+> The reason why we have a static type system is so that we can correctly make _assumptions_ about the data we work with.
+
+Scala generics were invented as an extension to this fact: by working with a `List[Int]`, for example, we can safely make the assumption that all this list will ever contain is integers. This will allow us, for instance, to sum up all the numbers in the list. This would not have been possible if we hadn't known the types of all elements.
+
+## 2. Why We Use Generics in Scala
+
+There is another big reason that Scala generics are so useful:
+
+> Generics allow us to reuse the same logic on many (potentially unrelated) types.
+
+Let's assume for a moment that you're designing your own data structure. Say, a list. We start simple, with a definition that will allow us to store integers, because in our application we work with numbers most of the time. A definition might look something like this:
+
+```scala
+ trait MyList {
+ def head: Int
+ def tail: MyList
+ }
+```
+
+The fundamental operations of a list are to
+- get the first element (head)
+- get the rest of the list, without the head (which is also a list)
+
+We can implement this list by considering two cases
+- an empty list, for which both the `head` and `tail` method return nothing/throw an exception
+- a non-empty list which contain both pieces of data
+
+Sample implementations would look like this:
+
+```scala
+ case class Empty() extends MyList {
+ override def head = throw new NoSuchElementException()
+ override def tail = throw new NoSuchElementException()
+ }
+
+ case class NonEmpty(h: Int, t: MyList) extends MyList {
+ override def head = h
+ override def tail = t
+ }
+```
+
+We have a list definition, so we can start using it:
+
+```scala
+val someNumbers: MyList = NonEmpty(1, NonEmpty(2, NonEmpty(3, Empty())))
+val secondNumber = someNumbers.tail.head
+```
+So far, so good.
+
+Let's imagine now that your application needs a list that would be applicable to Strings, too. In this case, our current list doesn't suffice, because it's strictly tied to integers. But we can copy it!
+
+```scala
+ trait MyListString {
+ def head: String
+ def tail: MyListString
+ }
+
+ case class EmptyString() extends MyListString {
+ override def head = throw new NoSuchElementException()
+ override def tail = throw new NoSuchElementException()
+ }
+
+ case class NonEmptyString(h: String, t: MyListString) extends MyListString {
+ override def head = h
+ override def tail = t
+ }
+```
+
+Done - one copy/paste and a few renames, and we also have a list of strings!
+
+```scala
+val someStrings: MyListString = NonEmptyString("I", NonEmptyString("love", NonEmptyString("Scala", EmptyString())))
+```
+
+But what if our needs increase? What if we want to apply this list concept to more than `Int` and `String` and copying becomes unsustainable? We can use one List with `Any` (the parent of all types) as the type!
+
+```scala
+ trait MyListAny {
+ def head: Any
+ def tail: MyListAny
+ }
+
+ case class EmptyAny() extends MyListAny {
+ override def head = throw new NoSuchElementException()
+ override def tail = throw new NoSuchElementException()
+ }
+
+ case class NonEmptyAny(h: Any, t: MyListAny) extends MyListAny {
+ override def head = h
+ override def tail = t
+ }
+```
+
+Now, we only need to use this one, and it will be applicable for both numbers and strings and everything else!
+
+However, there's a problem. Because the elements are of type `Any`, we can insert numbers, strings, and everything else _in the same list_, which means we cannot make any assumptions about the data actually stored within. Even if we did have a list of just numbers, because they're typed with `Any`, that will not allow us to do anything with those numbers (e.g. sum them or do any statistics).
+
+We've lost the very reason static typing exists (the ability to make assumptions about the data). In other words, we've lost _type safety_.
+
+## 3. How to Use Generics in Scala
+
+The best solution for this situation is to make the list data structure _generic_. Here's how we can add generics to our list:
+
+```scala
+ trait GoodList[A] {
+ def head: A
+ def tail: GoodList[A]
+ }
+```
+
+The `[A]` is the _type argument_ of the data structure. Once we add the type argument, we can use it anywhere in the body of the trait/class, and the compiler will always replace `A` with the concrete type we'll end up using. Let's continue the implementation of the generic subtypes of list, and we'll demonstrate shortly.
+
+```scala
+ case class GoodEmpty[A]() extends GoodList[A] {
+ override def head = throw new NoSuchElementException()
+ override def tail = throw new NoSuchElementException()
+ }
+
+ case class GoodNEmpty[A](h: A, t: GoodList[A]) extends GoodList[A] {
+ override def head: A = h
+ override def tail: GoodList[A] = t
+ }
+```
+
+Notice that the implementations have not changed aside from the types the methods need to return!
+
+Because we added the type argument to `GoodList`, the compiler will replace it with the real type we end up using:
+
+```scala
+val goodNumbers: GoodList[Int] = GoodNEmpty(1, GoodNEmpty(2, GoodNEmpty(3, GoodEmpty())))
+val firstNumber: Int = goodNumbers.head
+```
+
+In this case, we know for a fact that `firstNumber` must be an Int, by the definition of the `head` method, which returns the same type `A` (which is `Int` in this case) as the list was defined with.
+
+We get multiple benefits by using Scala generics in this way:
+- we eliminate the need to copy and paste
+- we can reuse the same code/logic for _all_ types
+- we guarantee type safety
+
+Now, generics in Scala lead to a whole range of [features](/scala-variance-positions/) and potentially [difficult concepts](/contravariance/), but here are some features of generics that you can use right now.
+
+First, you can add type arguments to every class or trait that you want to reuse for different types, in the style shown above.
+
+Second, if you need _multiple_ type arguments, you can do that too. For instance, if you want to define a "dictionary"/map data structure, you would write something like this:
+
+```scala
+ trait MyMap[K, V] {
+ def put(key: K, value: V): MyMap[K, V]
+ def get(key: K): V
+ }
+```
+
+and the mechanics would work in the same way:
+- after defining the type arguments `K, V` you can use them inside the trait body
+- the `put` method takes a key of exactly type `K` and a value of exactly type `V`, and returns a map of the same type (see type safety?)
+- the `get` method takes a key of type `K` and returns a value of type `V`, exactly the same types with which we defined the map
+
+Third, you can also reuse a single piece of logic (i.e. a method) by making it generic. For instance, if you want to find the last element of a list, you can write
+
+```scala
+ def lastElement[A](list: GoodList[A]): A =
+ if (list == GoodEmpty[A]()) throw new NoSuchElementException
+ else if (list.tail == GoodEmpty[A]()) list.head
+ else lastElement(list.tail)
+```
+
+and that would be a generic method in Scala.
+
+## 4. Conclusion
+
+This article was a slow and smooth introduction to Scala generics, for people who are getting started with Scala and particularly those coming from dynamically-typed languages like Python or JavaScript. We discussed why static typing is useful, how generics help us reuse code easily, and some generics features such as multiple type arguments and generic methods.
+
+If you liked this approach, we explain a lot more such core Scala concepts in the [Scala Essentials course](https://rockthejvm.com/p/scala), so consider checking it out.
diff --git a/_posts/2022-05-17-scala-3-type-projections.md b/_posts/2022-05-17-scala-3-type-projections.md
new file mode 100644
index 000000000000..b5c78f310e4e
--- /dev/null
+++ b/_posts/2022-05-17-scala-3-type-projections.md
@@ -0,0 +1,164 @@
+---
+title: "Scala 3 and General Type Projections"
+date: 2022-05-17
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, scala 3]
+excerpt: "Scala general type projections are unsound and were removed in Scala 3 - what do you mean?"
+---
+
+This article is about Scala 3. We've talked a lot about the additions in Scala 3 (which you can easily search on the blog), but it's also worth talking about the removals. In particular, this article will focus on the fact that "general type projections are unsound": what that phrase means, what that leads to, and why the feature was removed in Scala 3.
+
+This removal (along with dozens of other changes) was explained in depth in the [Scala 3 New Features](https://rockthejvm.com/p/scala-3-new-features) course.
+
+## 1. Background and Context
+
+We should know already that we can define classes, objects and traits _inside_ other classes, objects and traits.
+
+```scala
+class Outer {
+ class Inner
+}
+```
+
+In this example, each instance of `Outer` gives rise to a different `Inner`:
+
+```scala
+val o1 = new Outer
+val o2 = new Outer
+val i1 = new o1.Inner
+val i2 = new o2.Inner
+```
+
+The instances `o1` and `o2` result in the different _types_ `o1.Inner` and `o2.Inner`, and they're completely unrelated.
+
+```scala
+val i3: o1.Inner = new o2.Inner // compiler error (type mismatch)
+```
+
+However, as we explain in the [Advanced Scala course](https://rockthejvm.com/p/advanced-scala), all possible `o.Inner` types are subtypes of a general type called `Outer#Inner`. This is called a _type projection_, which is a pretty cool feature of Scala's type system.
+
+In Scala 2, it was also possible to express type projections based on types which were themselves abstract, i.e. abstract type members or generic type arguments. So it was possible to write something like `A#Inner`, where the compiler only knows that `A <: Outer`, for instance. This is called a "general" or "abstract" type projection, because the root `A` is not concrete.
+
+We used abstract type projections in the Scala 2 [type-level programming mini-series](/type-level-programming-part-1/) to force the compiler to make type resolutions at compile time, to a wonderful effect (sorting types at compile time)
+
+The problem is, it's not quite right. Martin Odersky initially signalled this by showing [an example](https://github.com/lampepfl/dotty/issues/1050) where the general type projections leads to uncompilable code which does compile and throws an error. The example does not compile in 2.13 anymore so the issue was fixed, but the general argument remains.
+
+## 2. Using General Type Projections
+
+I'll follow upon an exercise that I used in the [Advanced Scala 2 course](https://rockthejvm.com/p/scala-advanced-old) to practice path-dependent types and type projections. The exercise sounds like this — assume we'd like to build a general library for fetching type-safe fields from a database. We have a general type that describes items in the database, along with an identifier (key) in the folloing form:
+
+```scala
+ trait ItemLike {
+ type Key
+ }
+
+ trait Item[K] extends ItemLike {
+ type Key = K
+ }
+```
+
+We forced the type `Key` in `Item` to be exactly the same as the generic type argument `K`. We'd like to be able to define a method called `get`, such that we pass a `Key` as an argument, and return an _item type_ for which that `Key` was defined. The signature was the goal of the exercise. The goal was that, given some `Item` types such as
+
+```scala
+ class StringItem extends Item[String]
+ class IntItem extends Item[Int]
+```
+
+we would be able to say
+
+```scala
+get[IntItem](42) // ok, returns an IntItem
+get[StringItem]("Scala") // ok, returns a StringItem
+get[StringItem](55) // not ok, should not compile
+```
+
+The solution signature was this:
+
+```scala
+def get[I <: ItemLike](key: I#Key): I = ??? // implementation not important (and also impossible without some other info)
+```
+
+and lo and behold, the code compiles for those previous examples.
+
+## 3. Compiling Code that Breaks
+
+However, let me follow on the process to show you how quickly even this code can lead to trouble at runtime. Assume that we expand this suite of definitions with the following two types:
+
+```scala
+trait ItemAll extends ItemLike {
+ override type Key >: Any
+}
+
+trait ItemNothing extends ItemLike {
+ override type Key <: Nothing
+}
+```
+
+These type bounds don't really make sense, because
+- there's no supertype of `Any`
+- there's no subtype of `Nothing`
+
+However, the compiler allows setting these bounds (they're called "bad bounds" for obvious reasons) because the compiler allows setting bounds with respect to any type, as long as the bounds do reconcile in a concrete class. Of course, there is no class that is able to extend both types:
+
+```scala
+class ItemWeird extends ItemAll with ItemNothing // does not compile
+```
+
+However, there's nobody preventing us from "writing" the type `ItemAll with ItemNothing`, even though there's no possible real class that can conform to this type. Let us set up a few constructs that use general type projections and see how they can tie up with the impossibility of defining a value of type `ItemAll with ItemNothing`.
+
+We'll first define a method that returns an identity function, by virtue of a generic type argument which extends `ItemAll`:
+
+```scala
+def funcAll[I <: ItemAll]: Any => I#Key = x => x
+```
+
+This function compiles and is legal code because, given the fact that `I <: ItemAll`, then the type `I` surely has the type member `Key` which is a supertype of `Any` (because any change in the bounds would lead to uncompilable code), so it must be that `I#Key >: Any`, so the identity function `x => x` is legal and of the type `Any => I#Key`.
+
+We can also write a symmetrical function with respect to the Nothing type:
+
+```scala
+def funcNothing[I <: ItemNothing]: I#Key => Nothing = x => x
+```
+
+Similarly, if we know `I <: ItemNothing`, then we know that `I` has an abstract type `Key` which is a subtype of `Nothing`. Therefore, the identity function `x => x` works, because the argument `x` is of type `I#Key`, therefore the return value of the function belongs to `I#Key <: Nothing`, so the identity function conforms to the type `I#Key => Nothing`.
+
+Now for the truly evil part:
+
+```scala
+def funcWeird[I <: ItemAll with ItemNothing]: Any => Nothing =
+ funcAll[I].andThen(funcNothing[I])
+```
+
+This function is also legal. There's nobody preventing us from "using" the type `ItemAll with ItemNothing` even though there's no possible concrete type that conforms to it.
+1. Because `I <: ItemAll`, we can call `funcAll[I]`, which is an identity function of type `Any => I#Key`.
+2. Because `I <: ItemNothing`, we can call `funcNothing[I]`, which is an identity function of type `I#Key => Nothing`.
+3. Because the return type of `funcAll[I]` is the same as the argument type of `funcNothing[I]`, we can chain these two.
+
+The result is a compilable abomination which doesn't make sense, because we've made the compiler to delegate the bounds-checking phase to... never. The function we obtain is legal, and it will never work.
+
+Writing anything semi-legitimate in an application, such as
+
+```scala
+val anInt: Int = funcWeird("Scala")
+println(anInt + 1)
+```
+
+will run into...
+
+```txt
+Exception in thread "main" java.lang.ClassCastException: class java.lang.String cannot be cast to class scala.runtime.Nothing$ (java.lang.String is in module java.base of loader 'bootstrap'; scala.runtime.Nothing$ is in unnamed module of loader 'app')
+ at scala.Function1.$anonfun$andThen$1(Function1.scala:85)
+ at com.rockthejvm.part3removals.TypeProjections$.main(TypeProjections.scala:39)
+ at com.rockthejvm.part3removals.TypeProjections.main(TypeProjections.scala)
+```
+
+And of course it does! The identity function is of type `Any => Nothing`, which means the String we pass to this function will have to be converted to `Nothing`. It doesn't make sense, and the runtime catches up to us.
+
+## 4. Explanation and Conclusion
+
+The reason is the general type projection `I#Key`. Because the compiler allows us to write an abstract type projection, the compiler cannot do any bound compatibility checks on the `Key` member of `I` because `I` is abstract, and therefore it has no information on what `I#Key` can or cannot be.
+
+The phrase "general type projection is unsound" means that allowing this feature would lead to corner cases where the code should not compile, but it does, and leads to the kind of nonsense that we demonstrated earlier.
+
+This article wanted to show you how, and why, the feature of general type projections were removed in Scala 3, with and example and a piece of uncompilable code which does compile and runs into trouble.
diff --git a/_posts/2022-05-26-free-monad.md b/_posts/2022-05-26-free-monad.md
new file mode 100644
index 000000000000..24ceee89a2c2
--- /dev/null
+++ b/_posts/2022-05-26-free-monad.md
@@ -0,0 +1,534 @@
+---
+title: "Free Monad in Scala"
+date: 2022-05-26
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [abstract]
+excerpt: "A tutorial on the Free monad in Scala, how it works and what it's good for."
+---
+
+This article describes the Free monad in Scala — how it works, what it's good for and why we need it in the first place.
+
+This piece assumes you're already very comfortable with Scala. If you know a bit of Cats (which we teach in the [Cats course](https://rockthejvm.com)), that's a bonus, but not required for this article, because we're going to lay all the groundwork for what we need here.
+
+This post was written for Scala 3, but the concepts work almost identically for Scala 2 if you replace [givens with implicits](/givens-vs-implicits/).
+
+If you want to see this article in video form, please watch below:
+
+{% include video id="lzlCjgRWPDU" provider="youtube" %}
+
+## 1. Introduction
+
+In order to describe the Free monad, we need to understand what a Monad is. We've already written several approaches to it:
+
+- [a way to avoid cluttered code](/monads/)
+- [a type describing sequential computations](/another-take-on-monads/)
+- [a monoid in the category of endofunctors](/monads-are-monoids-in-the-category-of-endofunctors/)
+
+I encourage you to read at least the first two approaches. Monads are very powerful and form the building blocks for every useful piece of purely functional code. We describe monads as a type class, which takes a generic type argument which is itself generic (a higher-kinded type):
+
+```scala3
+trait Monad[M[_]] {
+ def pure[A](a: A): M[A]
+ def flatMap[A, B](ma: M[A])(f: A => M[B]): M[B]
+}
+```
+
+Monads describe, in essence, two fundamental operations:
+
+- a way to wrap a plain value into a "wrapper" type
+- a way to obtain a wrapper type from another, through a specific kind of transformation function
+
+Just from these two operations, we can describe an incredible variety of computations:
+
+- sequential collections, e.g. lists
+- potentially absent values, i.e. Options
+- computations that might fail, i.e. Try
+- potentially failed computations with an error type of our choosing, e.g. Either
+- computations that might perform side effects, i.e. IO from [Cats Effect](https://rockthejvm.com/p/cats-effect)
+- combinations thereof, e.g. ZIO, which take dependencies, can fail, and can perform side effects in the same computation
+
+The Free monad describes a similar "sequential" capability for a wrapper type _and for a well-defined value type_.
+
+## 2. Enter The Free Monad
+
+The Free monad in Scala can be described by a similar type signature, taking a type argument `M[_]` which is the container exhibiting monadic capabilities, _and_ a type argument `A` which is the value type with which that wrapper is defined. In other words, a Free monad trait can be written as
+
+```scala3
+trait Free[M[_], A]
+```
+
+Because it's a monad, we can also write the `pure` and `flatMap` methods, with the exact same meaning. However, since the Free data type has a slightly different signature, the `pure` and `flatMap` will also look a little different:
+
+```scala3
+trait Free[M[_], A] {
+ def pure(a: A): Free[M, A]
+ def flatMap[B](f: A => Free[M, B]): Free[M, B]
+}
+```
+
+A difference from the "regular" monad is that in this case, we shouldn't really need an instance of Free to build a new instance of Free with the `pure` method. So we'll move the `pure` method to a companion object, while keeping the `pure` concept intact:
+
+```scala3
+trait Free[M[_], A] {
+ def flatMap[B](f: A => Free[M, B]): Free[M, B]
+}
+
+object Free {
+ def pure[M[_], A](a: A): Free[M, A] = ???
+}
+```
+
+We'll leave the `pure` method unimplemented for now. The important bit is that we have the same `pure` + `flatMap` concepts here as well, even though the methods themselves are moved to different places. Now, because we have `pure` and `flatMap`, we can implement the `map` method for free — as we teach in the [Scala with Cats course](https://rockthejvm.com), the Functor's fundamental method can be expressed in terms of `pure` and `flatMap`:
+
+```scala3
+trait Free[M[_], A] {
+ // ... the rest of the code
+ import Free._
+ def map[B](f: A => B): Free[M, B] = flatMap(a => pure(f(a)))
+}
+```
+
+This `map` method will come in handy later.
+
+Besides `pure`, `flatMap` and `map` (which derives from the other two), we also have a function that turns a regular `M[A]` into a Free instance. This function is called "lift", either named `liftM` or `liftM` in real libraries, and looks like this:
+
+```scala3
+object Free {
+ // ... the rest of the code
+ def liftM[M, A](ma: M[A]): Free[M, A]
+}
+```
+
+That's it! This is the Free monad.
+
+But... why?
+
+Before we implement an actual Free monad, we'll take a detour to explain why the Free monad is a useful concept in the first place. We'll go through an example to understand how the Free monad would be actually used.
+
+## 3. Why We Need The Free monad
+
+### 3.1. Writing Programs with The Free Monad
+
+Let's imagine we're writing a small tool to interact with a custom database we have at work. For the sake of simplicity, we'll assume the fundamental operations of the database to be the regular CRUD:
+
+```scala3
+trait DBOps[A]
+case class Create[A](key: String, value: A) extends DBOps[Unit]
+case class Read[A](key: String) extends DBOps[A]
+case class Update[A](key: String, value: A) extends DBOps[A]
+case class Delete(key: String) extends DBOps[Unit]
+```
+
+This suite of operations (some of which can wrap others in real libraries) bears the fancy name of "algebra", for the reason that any expression composed out of these fundamental types through some operators (like nesting or regular methods) belong to this group as well.
+
+It's easy to imagine an example of an interaction with such a database:
+
+- read something from a key, like the name of a person
+- change it, e.g. run a `toUppercase` on it
+- associate this new value to another key
+- delete the old key
+
+This is a sequential suite of operations. Because it's sequential, a monadic data type to describe all of these operations can prove very useful. However, instead of writing a type class instance for `Monad[DBOps]` and following the [tagless-final](/tagless-final) approach, we're going to use a Free monad:
+
+```scala3
+type DBMonad[A] = Free[DBOps, A]
+```
+
+Because we're using a Free monad, instead of describing combinators of the data types above, we're going to "lift" them to Free through smart constructors:
+
+```scala3
+def create[A](key: String, value: A): DBMonad[Unit] =
+ Free.liftM[DBOps, Unit](Create(key, value))
+
+def get[A](key: String): DBMonad[A] =
+ Free.liftM[DBOps, A](Read[A](key))
+
+def update[A](key: String, value: A): DBMonad[A] =
+ Free.liftM[DBOps, A](Update[A](key, value))
+
+def delete(key: String): DBMonad[Unit] =
+ Free.liftM(Delete(key))
+```
+
+and with these smart constructors in place, we can immediately imagine an abstract program that does what we said above — read, change, create new entry, delete old entry:
+
+```scala3
+ def myLittleProgram: DBMonad[Unit] = for {
+ _ <- create[String]("123-456", "Daniel")
+ name <- get[String]("123-456")
+ _ <- create[String]("567", name.toUpperCase())
+ _ <- delete("123-456")
+ } yield ()
+```
+
+This program can be completely described in terms of the Free monad, regardless of what wrapper type we use. The only problem is that it's a _description_ of a computation; it does not perform any meaningful work in the world, like, you know, interacting with an actual database.
+
+We need something to _interpret_ this program.
+
+### 3.2. Free Monad FoldMap
+
+What does "interpretation" even mean? Interpreting a program means transforming this abstract program written in terms of the Free monad into another data type that actually performs the computations when evaluated. A good candidate for such a data structure is, for example, the Cats Effect IO. However, we can pick another data type of our choosing, with the meaning of
+
+- potentially absent values: Option
+- potentially failed computations: Try
+- potentially failed computations with an error type of our choosing: Either
+- computations that can perform side effects: IO
+- combinations of the above: ZIO
+- aggregating values: List
+
+This is why the Free monad has another operation that can "evaluate" an instance of Free to one of these data types. The operation is called `foldMap` and looks like this:
+
+```scala3
+trait Free[M[_], A] {
+ // ... existing code
+ def foldMap[G[_]: Monad](natTrans: M ~> G): G[A]
+}
+```
+
+This one is a bit more complicated. Let's take each piece in turn.
+
+First of all, what's a `natTrans` and the `~>` symbol? The concept of "natural transformation" is a higher-kinded Function1 type that looks like this:
+
+```scala3
+ trait ~>[F[_], G[_]] {
+ def apply[A](fa: F[A]): G[A]
+ }
+```
+
+So instead of a regular Function1 taking value types as type parameters, now we operate at a higher kind. We used this concept in our demonstration for why [monads are monoids in the category of endofunctors](/monads-are-monoids-in-the-category-of-endofunctors/). Examples of natural transformations in real life include:
+
+- `Try[A].toOption`: this is an example of an implementation of a natural transformation between `Try` and `Option`
+- `List[A].headOption` which returns the head of the list, if it exists: an example of an implementation of a natural transformation between `List` and `Option`
+- `Option[A].toList`: the reverse
+
+We can abstract away this concept of natural transformation by the `~>` symbol, which looks like a function type at a higher kind.
+
+Back to `foldMap`. Notice that the return value of `foldMap` is `G[A]`, so a different monadic type than `M[_]`, assuming that G "is" a Monad, by the context bound `G[_]: Monad`. This is important, because the evaluation of an instance of Free can only happen if the wrapper type which we're evaluating _to_ is also a monad, i.e. exhibits monadic behavior.
+
+### 3.3. Natural Transformations
+
+For our little program, we can interpret it with `foldMap`, if we can create a natural transformation from our type `DBOps` to some other type that will actually perform the actions described by the program. Let's write a simple IO data type in the style of Cats Effect:
+
+```scala3
+case class IO[A](unsafeRun: () => A)
+object IO {
+ def create[A](a: => A): IO[A] = IO(() => a)
+}
+```
+
+The IO type encapsulates computations that evaluate to `A`, with potential side effects. The IO data type is a monad, meaning that we can create a Monad instance for it:
+
+```scala3
+given ioMonad: Monad[IO] with {
+ override def pure[A](a: A) =
+ IO(() => a)
+ override def flatMap[A, B](ma: IO[A])(f: A => IO[B]) =
+ IO(() => f(ma.unsafeRun()).unsafeRun())
+}
+```
+
+Being a monad, the IO data type is a suitable "evaluator" for our abstract program. The only piece that we need is a natural transformation from `DBOps` to IO. This natural transformation is the logic of evaluating the abstract CRUD operations (for now just data structures) into actual effects in the real world. For that, we'll need an actual database to store and retrieve data. For this simple example, we'll assume a simple mutable map, but this can easily be replaced in practice by a real database.
+
+```scala3
+val myDB: mutable.Map[String, String] = mutable.Map()
+```
+
+For _our_ particular database, in order to read/write values of type `A`, we'll also provide some awesome serialization/deserialization API to/from String:
+
+```scala3
+def serialize[A](a: A): String = a.toString
+def deserialize[A](value: String): A = value.asInstanceOf[A]
+```
+
+Of course, with your own database, the communication protocol will be different, but for this example it'll be sufficient.
+
+The natural transformation from `DBOps` to our IO data type will need to take the "database" and the "protocol" into account. All we need to do is convert all cases of `DBOps` into proper `IO`s that will perform effects on the database when evaluated, so we can resort to simple pattern matching for this:
+
+```scala3
+val dbOps2IO: DBOps ~> IO = new (DBOps ~> IO) {
+ override def apply[A](fa: DBOps[A]): IO[A] = fa match {
+ case Create(key, value) => IO.create {
+ // database insert query - here, just printing
+ println(s"insert into people(id, name) values ($key, $value)")
+ myDB += (key -> serialize(value))
+ ()
+ }
+ case Read(key) => IO.create {
+ println(s"select * from people where id=$key limit 1")
+ deserialize(myDB(key))
+ }
+ case Update(key, value) => IO.create {
+ println(s"update people(name=$value) where id=$key")
+ val oldValue = myDB(key)
+ myDB += (key -> serialize(value))
+ deserialize(oldValue)
+ }
+ case Delete(key) => IO.create {
+ println(s"delete from people where id=$key")
+ ()
+ }
+ }
+}
+```
+
+### 3.4. Evaluating the Program
+
+With all pieces in place, we can now run the interpreter of our abstract program with `foldMap` and with the natural transformation we've just implemented:
+
+```scala3
+val ioProgram: IO[Unit] = myLittleProgram.foldMap(dbOps2IO)
+```
+
+This is a single `IO[Unit]` effect which can be passed around, reused, or "run" in main:
+
+```scala3
+def main(args: Array[String]): Unit = {
+ ioProgram.unsafeRun() // PERFORMS THE ACTUAL WORK
+}
+```
+
+This will execute the IO effect, which in turn will evaluate our abstract program.
+
+### 3.5. The Free Monad Pattern
+
+Why have we gone all this way just to insert a bunch of data into a database?
+
+The Free monad is a pattern which allows us to separate
+
+- the description of fundamental operations
+- the business logic of our application
+- the evaluation of that business logic
+
+In other words, our abstract program is what matters for our application/business logic. We can keep that fixed, and give it different interpreters depending on how our requirements change — e.g. perhaps we want the program to be evaluated asynchronously — or we can do the reverse. This makes it very easy to maintain, because we can work independently on either
+
+- the business logic, while keeping interpreters fixed
+- the interpreters, while keeping the business logic fixed
+- the fundamental operations and the interpreter(s), while keeping the business logic fixed
+
+So notice the **flexibility** we get by choosing to work on a piece of the system without affecting the others. Another benefit of this approach is **testability**, because we can always supply a "testing" monad to evaluate the program, make assertions and ensure the business logic is correct, while the interpreter itself can be independently tested.
+
+## 4. Implementing a Free Monad
+
+You might have noticed that our current code is incomplete. We took a detour to understand the reasons why Free monads are useful and how they help with code decoupling. Let's move back to the Free monad itself, because our Free type currently looks like this:
+
+```scala3
+trait Free[M[_], A] {
+ def flatMap[B](f: A => Free[M, B]): Free[M, B]
+ def map[B](f: A => B): Free[M, B] = flatMap(a => pure(f(a)))
+ def foldMap[G[_]: Monad](natTrans: M ~> G): G[A]
+}
+
+object Free {
+ def pure[M[_], A](a: A): Free[M, A] = ???
+ def liftM[M[_], A](ma: M[A]): Free[M, A] = ???
+}
+```
+
+We're going to write actual instances of Free as case classes for all fundamental operations:
+
+- pure
+- flatMap
+- liftM
+
+```scala3
+object Free {
+ // ... existing code
+ case class Pure[M[_], A](a: A) extends Free[M, A]
+ case class FlatMap[M[_],A,B](fa: Free[M, A], f: A => Free[M, B]) extends Free[M, B]
+ case class Suspend[M[_], A](ma: M[A]) extends Free[M, A]
+}
+```
+
+The `Pure` type simply wraps a single value of type `A`; we'll make use of this later when we evaluate `foldMap`. The `FlatMap` case class follows the signature of the `flatMap` method in the Free trait, keeping the Free instance to be transformed and a function to turn an `A` into another instance of `Free[M, B]`. The final case class is `Suspend` which corresponds to the `liftM` method.
+
+With these case classes in place, we can evaluate `pure`, `flatMap` and `liftM` immediately:
+
+```scala3
+trait Free[M[_], A] {
+ import Free.*
+ def flatMap[B](f: A => Free[M, B]): Free[M, B] = FlatMap(this, f) // added now
+ def map[B](f: A => B): Free[M, B] = flatMap(a => pure(f(a)))
+ def foldMap[G[_]: Monad](natTrans: M ~> G): G[A]
+}
+
+object Free {
+ def pure[M[_], A](a: A): Free[M, A] = Pure(a) // added now
+ def liftM[M[_], A](ma: M[A]): Free[M, A] = Suspend(ma) // added now
+
+ // added earlier
+ case class Pure[M[_], A](a: A) extends Free[M, A]
+ case class FlatMap[M[_],A,B](fa: Free[M, A], f: A => Free[M, B]) extends Free[M, B]
+ case class Suspend[M[_], A](ma: M[A]) extends Free[M, A]
+}
+```
+
+So the final piece that we need to implement is `foldMap`. The `foldMap` method will now need to evaluate this Free instance, depending on which type this instance belongs to:
+
+- If this instance is a `Pure`, then return a "pure" `G[A]`. Because `G` is a monad, we can do that easily.
+- If this instance is a `Suspend`, then take the `M[A]` wrapped inside and turn that into a `G[A]` because we have the natural transformation handy.
+- If this instance is a `FlatMap`, then this instance has two fields: another Free instance and a transformation function.
+ - Run a recursive `foldMap` on the wrapped Free instance, returning a `G[A]`.
+ - Then `flatMap` that because `G` is a monad, so we can use the Monad instance to do it.
+
+Notice how the presence of a given/implicit `Monad[G]` in scope is crucial here. Our code for `foldMap` is as follows:
+
+```scala3
+def foldMap[G[_]](natTrans: M ~> G)(using monadG: Monad[G]): G[A] = this match {
+ case Pure(a) => Monad[G].pure(a)
+ case Suspend(ma) => natTrans.apply(ma)
+ case FlatMap(fa, f) =>
+ monadG.flatMap(fa.foldMap(natTrans))(a => f(a).foldMap(natTrans))
+}
+```
+
+Here, we've slightly changed the signature of `foldMap` so that we can make use of the given `Monad[G]`, but we can also keep the old signature and add a summoning method for Monad:
+
+```scala3
+// alternative
+object Monad {
+ def apply[M[_]](using monad: Monad[M]): Monad[M] = monad
+}
+
+trait Free[M[_], A] {
+ // ... existing code
+
+ // added now
+ def foldMap[G[_]: Monad](natTrans: M ~> G): G[A] = this match {
+ case Pure(a) => Monad[G].pure(a)
+ case Suspend(ma) => natTrans.apply(ma)
+ case FlatMap(fa, f) => // need a G[B]
+ Monad[G].flatMap(fa.foldMap(natTrans))(a => f(a).foldMap(natTrans))
+ }
+}
+```
+
+### 5. Final Code
+
+The full program with all pieces put together looks like this:
+
+```scala3
+object FreeMonad {
+ trait Monad[M[_]] {
+ def pure[A](a: A): M[A]
+ def flatMap[A, B](ma: M[A])(f: A => M[B]): M[B]
+ }
+
+ object Monad {
+ def apply[M[_]](using monad: Monad[M]): Monad[M] = monad
+ }
+
+ trait ~>[F[_], G[_]] {
+ def apply[A](fa: F[A]): G[A]
+ }
+
+ trait Free[M[_], A] {
+ import Free.*
+ def flatMap[B](f: A => Free[M, B]): Free[M, B] = FlatMap(this, f)
+ def map[B](f: A => B): Free[M, B] = flatMap(a => pure(f(a)))
+ def foldMap[G[_]: Monad](natTrans: M ~> G): G[A] = this match {
+ case Pure(a) => Monad[G].pure(a)
+ case Suspend(ma) => natTrans.apply(ma)
+ case FlatMap(fa, f) => // need a G[B]
+ Monad[G].flatMap(fa.foldMap(natTrans))(a => f(a) .foldMap(natTrans) )
+ }
+ }
+
+ object Free {
+ def pure[M[_], A](a: A): Free[M, A] = Pure(a)
+ def liftM[M[_], A](ma: M[A]): Free[M, A] = Suspend(ma)
+
+ case class Pure[M[_], A](a: A) extends Free[M, A]
+ case class FlatMap[M[_],A,B](fa: Free[M, A], f: A => Free[M, B]) extends Free[M, B]
+ case class Suspend[M[_], A](ma: M[A]) extends Free[M, A]
+ }
+
+ // sequence computations as data structures, THEN attach the monadic type at the end
+ // "algebra"
+ trait DBOps[A]
+ case class Create[A](key: String, value: A) extends DBOps[Unit]
+ case class Read[A](key: String) extends DBOps[A]
+ case class Update[A](key: String, value: A) extends DBOps[A]
+ case class Delete(key: String) extends DBOps[Unit]
+
+ // definitions - fancier algebra
+ type DBMonad[A] = Free[DBOps, A]
+
+ // "smart" constructors
+ def create[A](key: String, value: A): DBMonad[Unit] =
+ Free.liftM[DBOps, Unit](Create(key, value))
+
+ def get[A](key: String): DBMonad[A] =
+ Free.liftM[DBOps, A](Read[A](key))
+
+ def update[A](key: String, value: A): DBMonad[A] =
+ Free.liftM[DBOps, A](Update[A](key, value))
+
+ def delete(key: String): DBMonad[Unit] =
+ Free.liftM(Delete(key))
+
+ // business logic is FIXED
+ def myLittleProgram: DBMonad[Unit] = for { // monadic
+ _ <- create[String]("123-456", "Daniel")
+ name <- get[String]("123-456")
+ _ <- create[String]("567", name.toUpperCase())
+ _ <- delete("123-456")
+ } yield () // description of a computation
+
+ // evaluate the program - interpreter/"compiler"
+ // IO
+ case class IO[A](unsafeRun: () => A)
+ object IO {
+ def create[A](a: => A): IO[A] = IO(() => a)
+ }
+
+ given ioMonad: Monad[IO] with {
+ override def pure[A](a: A) = IO(() => a)
+ override def flatMap[A, B](ma: IO[A])(f: A => IO[B]) =
+ IO(() => f(ma.unsafeRun()).unsafeRun())
+ }
+
+ val myDB: mutable.Map[String, String] = mutable.Map()
+ // TODO replace these with some real serialization
+ def serialize[A](a: A): String = a.toString
+ def deserialize[A](value: String): A = value.asInstanceOf[A]
+
+ // nat trans DBOps -> IO
+ val dbOps2IO: DBOps ~> IO = new (DBOps ~> IO) {
+ override def apply[A](fa: DBOps[A]): IO[A] = fa match {
+ case Create(key, value) => IO.create { // actual code that uses the database
+ println(s"insert into people(id, name) values ($key, $value)")
+ myDB += (key -> serialize(value))
+ ()
+ }
+ case Read(key) => IO.create {
+ println(s"select * from people where id=$key limit 1")
+ deserialize(myDB(key))
+ }
+ case Update(key, value) => IO.create {
+ println(s"update people(name=$value) where id=$key")
+ val oldValue = myDB(key)
+ myDB += (key -> serialize(value))
+ deserialize(oldValue)
+ }
+ case Delete(key) => IO.create {
+ println(s"delete from people where id=$key")
+ ()
+ }
+ }
+ }
+
+ val ioProgram: IO[Unit] = myLittleProgram.foldMap(dbOps2IO)
+
+ def main(args: Array[String]): Unit = {
+ ioProgram.unsafeRun() // PERFORMS THE ACTUAL WORK
+ }
+}
+```
+
+
+## 6. Conclusion
+
+In this article, we did a comprehensive overview of what a Free monad is, why it's useful, how it helps us decouple our code, how it adds flexibility and testability in our programs, and we implemented our own simple version of a Free monad along with an example abstract program and an interpreter for it in the style of Cats Effect.
+
+Use it well!
diff --git a/_posts/2022-06-02-scala-option.md b/_posts/2022-06-02-scala-option.md
new file mode 100644
index 000000000000..eb97cfb00844
--- /dev/null
+++ b/_posts/2022-06-02-scala-option.md
@@ -0,0 +1,235 @@
+---
+title: "Scala Option: A Gentle Introduction"
+date: 2022-06-02
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, beginners]
+excerpt: "Scala Options are some of the first things we learn - but why? What do they do, and why are they useful?"
+---
+
+This is a beginner-friendly article. We'll introduce Scala Options, a seemingly contrived data structure, but we'll see in this post that it's extremely powerful and useful in its simplicity. If you're just getting started with Scala and Options seem hard to understand, this article is for you.
+
+We teach Options in-depth in the [Scala Essentials](https://rockthejvm.com/p/scala) course, along with examples and exercises, if you want to check it out. In this article, we'll introduce Options, show you how they work, and _why_ they exist.
+
+
+If you want to see this article in video form, please watch below:
+
+{% include video id="xywCiwNwPEU" provider="youtube" %}
+
+## 1. Introduction to Scala Option
+
+The first real data structure we learn when getting started with Scala is the list. It's quick to set up, and easy to understand.
+
+```scala
+val aList: List[Int] = List(1,2,3)
+```
+
+You probably know by now that lists are immutable — every transformation on a list results in a new list — and there are some transformations we use a lot: `map`, `flatMap` and `filter`:
+
+```scala
+val aTransformedList = aList.map(x => x + 1) // [2,3,4]
+val aTransformedList_v2 = aList.flatMap(x => List(x, x+1)) // [1,2 ,2,3, 3,4]
+val aFilteredList = aList.filter(x => x % 2 == 0) // [2]
+```
+
+To recap:
+
+- `map` turns a list into another, where for every element in the original list, the supplied function is apply
+- `flatMap` runs a function on every element on the list, resulting in many mini-lists; these lists are then concatenated into the final result
+- `filter` keeps the elements which satisfy the boolean predicate passed as argument
+
+Because we have `map` and `flatMap`, we can run for-comprehensions on lists:
+
+```scala
+val combinedLists = for {
+ num <- List(1,2,3)
+ char <- List('a','b')
+} yield s"$num-$char"
+// List(1,2,3).flatMap(num => List('a','b').map(char => s"$num-$char"))
+// [1-a, 1-b, 2-a, 2-b, 3-a, 3-b]
+```
+
+You also probably know that for-comprehensions are rewritten by the compiler into chains of `map` and `flatMap`, as you can see in the above snippet. The coolest thing about for-comprehensions is that any data structure can be eligible for for-comprehensions, as long as they have `map` and `flatMap` with similar signatures as that of the `List` type.
+
+Why did we start this post describing lists instead of Options?
+
+The easiest way to understand a Scala Option is to imagine it as a **list with at most one value**, that is, it either contains a single value, or nothing at all.
+
+```scala
+val anOption: Option[Int] = Option(42) // an option that contains a single value: 42
+val anEmptyOption: Option[Int] = Option.empty // an option that doesn't contain any value
+```
+
+Let's see some operators on Options.
+
+## 2. Scala Option API
+
+First, let's talk about the possible cases of an Option:
+
+- an empty Option containing no value
+- an Option containing a single value
+
+We can build these cases explicitly:
+
+```scala
+val anOption_v2: Option[Int] = Some(42) // same as Option(42)
+val anEmptyOption_v2: Option[Int] = None // same as Option.empty
+```
+
+A very interesting case is when we use `Option(null)`, which gives back a `None`, i.e. an empty Option. We'll come back to this idea, because it leads to the main reason why Scala Options are useful in the first place.
+
+Much like lists, Options can be transformed with `map`, `flatMap` and `filter`. The `map` function transforms an Option by returning a new Option containing
+- nothing, if the original Option was empty
+- the transformed element of the original Option contained a value
+
+```scala
+val aTransformedOption: Option[Int] = anOption.map(x => x * 10) // Some(420)
+```
+
+The `flatMap` method is a bit more difficult, but it resembles the list `flatMap` method — the function we pass as argument turns an element into another Option:
+
+```scala
+val aTransformedOption_v2: Option[Int] = anOption.flatMap(x => Option(x * 10)) // Some(420)
+```
+
+However, in this case, the `flatMap` method doesn't have the _meaning_ of concatenation (as was the case for lists), because here there's either a single value, or nothing. For Option, the `flatMap` method is a way of _chaining_ computations that may either produce a value or nothing. This general idea of sequencing computations is [immensely powerful](/monads/).
+
+Finally, the `filter` method retains the original value of the Option if it passes a predicate, or it discards it otherwise, turning the original Option into None. Of course, if the original Option was empty to begin with, there's nothing to filter, hence nothing to process.
+
+```scala
+val aFilteredOption: Option[Int] = anOption.filter(x => x > 100) // None
+```
+
+Now, because we have `map` and `flatMap`, we also have for-comprehensions unlocked for Option:
+
+```scala
+val combinedOptions = for {
+ num <- Option(42)
+ str <- Option("Scala")
+} yield s"$str is the meaning of life, aka $num"
+```
+
+For those of you coming to Scala from another language, the for-comprehension might look a little... unusual. When our background is mostly imperative languages (Java, Python, C++, C#, etc), we tend to think about `for` structures as "loops". In Scala, a for-comprehension is **not** a loop, but rather a chain of `map` and `flatMap`, which we saw are expressions returning other data structures.
+
+Besides `map`, `flatMap` and `filter`, Options also have other APIs to transform values or run checks:
+
+```scala
+val checkEmpty: Boolean = anOption.isEmpty
+val innerValue: Int = anOption.getOrElse(99)
+val aChainedOption: Option[Int] = anEmptyOption.orElse(anOption)
+```
+
+The `isEmpty` method checks whether an Option contains nothing. The `getOrElse` method returns either the value inside the option, or the default value which we pass as argument. Finally, the `orElse` method returns the original Option if non-empty, or the other option otherwise. There's also a `get` method on Option, which returns the value inside, but beware: if the Option is empty, the method will crash!
+
+There are more methods in the Option data type, but these are the most frequently used.
+
+## 3. Why We Need Options in Scala
+
+At this point, you might be thinking why on Earth we need this weird data structure which resembles a list, but it only contains a single value or nothing. Why can't we just use a list?
+
+The reason why Options are useful is not the same why Lists are useful. Options are not used to "store" values — as your intuition clearly can tell, we can use lists to store as many values as we like. The utility of Options is not in data storage.
+
+Options are used to describe the **possible absence of a value**.
+
+If, like most of us, your experience is mostly with the "mainstream" languages like Java or C, you've probably been dealing with the absence of values by using the `null` reference. Because `null` is a proper replacement for every type, at the slightest suspicion that an expression might return null, we need to write defensive code so that we stay away from the dreaded `NullPointerException`.
+
+```scala
+def unsafeMethod(arg: Int): String = null // implementation not important, assume that for some code path the function returns null
+// defensive code
+val potentialValue = unsafeMethod(44)
+val myRealResult =
+ if (potentialValue == null)
+ "ERROR"
+ else
+ potentialValue.toUpperCase()
+```
+
+However, in real life we deal not just with one value, but with many. Assume we want to combine two such calls to `unsafeMethod`. How would we do it? Defensively, of course:
+
+```scala
+val callToExternalService = unsafeMethod(55)
+val combinedResult =
+ if (potentialValue == null)
+ if (callToExternalService == null)
+ "ERROR 1"
+ else
+ "ERROR 2"
+ else
+ if (callToExternalService == null)
+ "ERROR 3"
+ else
+ potentialValue.toUpperCase() + callToExternalService.toUpperCase()
+```
+
+Nothing short of horrible. What if we need to combine 3 values? Do we add 8 if-clauses there? This is unproductive, unmaintainable and unreadable.
+
+Options can save us from ourselves:
+
+```scala
+val betterCombinedResult: Option[String] = for {
+ firstValue <- Option(unsafeMethod(44))
+ secondValue <- Option(unsafeMethod(55))
+} yield firstValue.toUpperCase() + secondValue.toUpperCase()
+
+val finalValue = betterCombinedResult.getOrElse("ERROR")
+```
+
+The for-comprehension, because it's expressed in terms of `map` and `flatMap`, can take care automatically to check whether the Options inside are empty or not, and return the appropriate value, without us checking everything ourselves. This code offers many benefits:
+
+- much easier to understand: combine 2 Options if they're non-empty, return empty otherwise
+- easier to maintain: can easily add another Option there without blowing up the if-else conditions
+- much shorter, saving us time and (mental) space
+
+With this in mind, we'll write a first piece of good practice:
+
+> When you have APIs that risk returning nulls, wrap them in Options when you use them. Better yet, if you can change the API, return Option[YourType] instead.
+
+## 4. Resisting the Temptation to Retrieve Option Values
+
+Beginning Scala programmers, upon learning about Options, can successfully navigate Option transformations in other people's code. However, when some external code hands them an Option, the immediate itch is to try to "get" the value inside the Option, so that you can do something meaningful to it.
+
+That's a mistake.
+
+Because you don't know whether an Option is empty or not, if we ever want to "get" a value safely we need to test whether the Option is empty or not. Considering the same example with calling the `unsafeMethod` multiple times (this time wrapped inside Options), scratching the itch to "get" the values inside will lead us to code that looks like this:
+
+```scala
+val firstValue = Option(unsafeMethod(44))
+val secondValue = Option(unsafeMethod(55))
+val combinedResult_v2 =
+ if (firstValue.isEmpty)
+ if (secondValue.isEmpty)
+ "ERROR 1"
+ else
+ "ERROR 2"
+ else
+ if (secondValue.isEmpty)
+ "ERROR 3"
+ else
+ firstValue.get.toUpperCase() + secondValue.get.toUpperCase()
+```
+
+Notice something? This is just as bad as dealing with nulls in the first place!
+
+So here's the second piece of advice:
+
+> Resist the temptation to "get" the values inside your Options. Rather, use transformations like `map`, `flatMap`, `filter`, `orElse`, etc.
+
+## 5. The Option Mistake of Doom
+
+We mentioned at the beginning that we can build Options with the `Some` and `None` constructors explicitly. That's discouraged; you should always use the `Option` constructor/apply method all the time.
+
+The reason is that by the contract of `Some`, this Option is guaranteed to never contain a null value. However, if you ever write
+
+```scala
+val myOption = Some(myExpression)
+```
+
+on the grounds that you're "sure" the expression will never return null, you can be mistaken. If so, you'll be _gravely_ mistaken, because subsequent computations will rely on the fact that the Option is non-empty, or that the value contained is not null. If you ever break that contract, your application can crash so bad and so much later than the moment you wrote this code, that debugging can take hours, days or more.
+
+So here's the final bit of best practice:
+
+> Always use the `Option` "constructor"/apply method to build new Options. NEVER EVER use `Some(null)` or `Some(anExpressionThatCanBeNull)`, because you'll kill us all.
+
+## 6. Conclusion
+
+In this article, we introduced Scala Options: how you can start thinking about them, the main API, how to transform Options, _why_ they are useful, and we also shared some bits of best practice when dealing with them.
diff --git a/_posts/2022-06-06-roadmap-to-successful-data-engineer.md b/_posts/2022-06-06-roadmap-to-successful-data-engineer.md
new file mode 100644
index 000000000000..069ff6467767
--- /dev/null
+++ b/_posts/2022-06-06-roadmap-to-successful-data-engineer.md
@@ -0,0 +1,112 @@
+---
+title: "Roadmap to a Successful Data Engineer"
+date: 2022-06-06
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, beginners]
+excerpt: "One of the most prominent Rock the JVM students shares his insights to a successful career in Data Engineering."
+---
+
+_This article is brought to you by [Anirban Goswami](https://www.linkedin.com/in/anigos/) a.k.a. Ani, one of the earliest Rock the JVM students and fans, now a big data architect. Here, Ani shares his stories and tips for aspiring data engineers. Ani is mega-passionate about what he's doing, which is reflected in the slightly different, more conversational style of this article._
+
+> “Data! Data! Data! I can’t make bricks without clay!” - Arthur Conan Doyle
+
+Everything in this world starts from a purpose, a motivation which drives us to become better in any field. You can be an artist, a guitar player, a swimmer, a carpenter or an engineer who is working towards your success. Being a data engineer for more than a decade, I want to share my realization, thinking that it might help other aspirants to get some boost and acceleration to become the best version of themselves.
+
+## What is Data Engineering?
+
+I believe there will be a lot of people who are reading this article who are not 100% sure what a data engineer means. 10 years back, there was no such thing as "data engineer". Let us go back in history and see the revolution.
+
+First, not every coder is a Data Engineer! When someone graduates from college learning programming, packaging and deployment of applications, they are trained as software engineers; but you cannot become a data engineer coming out of college with the same path as an architect in software engineering. A Data Engineering mind gets built up over time. There is no shortcut to it but there are processes, there are principles and methodologies to become one — a true Data Engineer!
+
+Data and its importance became relevant to organizations around the world in the early 90s and since then organizations like Oracle, IBM, Microsoft, Teradata etc. invested a significant amount of resources — both financial and scientists — to create beautiful products for data consumers. Oracle DB, IBM DB2, Microsoft SQL Server, Teradata DB were the pioneers in that sector, and they are still relevant. Most of the time, language developers or analysts interact with these systems with Structured Query Language a.k.a. SQL. Most of the engineering work was already done by the vendor companies. Kudos to Oracle, MS, IBM for their tremendous optimization work, tooling and accelerators they have built for programming communities. These databases have been backing countless web applications, analytics and decision-making systems for 3+ decades now. Invaluable contribution, isn’t it?
+
+The above paragraph explained how the databases came about and how they revolutionized the data world. With all the good things they brought to the table, they introduced the most difficult problem to tackle - “Hunger for more”! Success is very addictive in nature, especially in engineering. Starting from the discovery to fire, wheel, motor and spaceship it has never stopped, only accelerated. The same is true for the data space.
+
+## Core Aspects of Big Data: Volume, Velocity, Variety
+
+There was a point in time — for example, when we needed tens of databases in an organization — when it was obvious that data volume would increase and that happened quickly. Big data is about **volume**. Volumes of data that can reach unprecedented heights. It’s estimated that _10 quintillion_ bytes of data is created each day, and as a result, there will be _180 zettabytes_ of data created by 2025 – which highlights an increase of 1000 times from 2005. As a result, it is now not uncommon for large companies to have Terabytes – and even Petabytes – of data in storage devices and on servers. This data helps to shape the future of a company and its actions, all while tracking progress.
+
+The growth of data, and the resulting importance of it, has changed the way we see data. There once was a time when we couldn't see the importance of data in the corporate world, but with the change of how we gather it, we’ve come to rely on it day to day. **Velocity** measures how fast the data is coming in. Some data will come in real-time, whereas others will come in fits and starts, sent to us in batches. And as not all platforms will experience the incoming data at the same pace, it’s important not to generalize, discount, or jump to conclusions without having all the facts and figures.
+
+**Data formats** are changing every day. Once there was a time when data format was flat file, say CSV, then we saw excel, TSVs, JSON, XML, AVRO, Parquet, ORC etc. Data is not just structured ones, today we generate more unstructured data than structured ones.
+
+Tackling the 3Vs of Big Data involved the first use case of the conventional data systems or the _Relational databases_. The architecture of such RDBMS systems is following one trend that tightly couples storage and compute systems. Use cases around the globe were never similar in practice, e.g. that you will add 100TB storage and that 100TB hardware will be used all the time for compute. Hadoop came in the late 90s to manage big data at scale, and they made the storage and compute _separate_. Since then Hadoop managed it with style, and we got several services such as MapReduce, Pig, Hive, Spark and mode. Today there is no better compute framework than Spark, which is cheap and efficient.
+
+All these technologies around big data require varied skill sets which are not restricted to just writing SQLs or scripting. There are multiple arguments that can come to support the likes of cloud DWs (data warehouses) where just knowing a few admin commands and writing SQLs will help you survive in this competitive market but that requires a tremendous amount of money to support them, and you are locked-in with their features. A data engineer is a technologist who is a blend of many skill-sets borrowed from different streams. A real data engineer is a software engineer to the core, a programmer at heart, an excellent SQL developer, a hardware optimizer and more. Let us explore and understand what we really need to excel at, so we can be hailed as a Data Engineer!
+
+## The Core strengths of a Data Engineer
+
+A data engineer’s life is not easy. When a web application fails to render data due to slowness in the back end database, a data engineer gets a call at night. If a business user is stuck getting data from a streaming system and waiting for the queue to get cleared up, they look for a data engineer. An ETL job overrunning and crossing SLAs (service level agreements), someone will wake up the data engineer from a good sleep. Collectively a data engineer should possess some of the best virtues of a solid engineering mind.
+
+**Problem solving**: Problems should excite a data engineer, be it performance issues, governance issues, access issues, memory leaks or even data modeling. We get them every day, and we should enjoy them and take them as a challenge, instead of running away from them. Problems and their solutions will mold you into a different breed. The more you live in this mindset, the richer your solutions will be and people will see that coming out of you when you speak, when you suggest, when you design.
+
+**Optimization**: Systems are built by humans. For ages, we've been creating systems which pretended to be optimal in nature but every time they failed us. This is so true even with all the modern day data engineering frameworks and platforms. They are optimal to some extent, but in the end they need a data engineer to push their limits. There's no better example than Apache Spark. It does so many things on its own but not everything. When it surrenders to the limitation, then the job of a great data engineer starts.
+
+**Anticipation**: When you design a solution, you should also try to anticipate the good (strengths) and the bad (limitations). It gives you confidence, it builds your design thinking. Think one step further: what can ruin the dream? What could crash this solution?
+
+**Balance - Tech or Functional**: At the end of the day we are implementing functionality (in terms of business) and all the functionalities are possible with technical abilities (raw programming skills). Understand the problem first and become a good listener to win. The more you understand the problem statement, the better you can balance.
+
+**Simplification**: Whatever it may take to sacrifice, do not over-engineer a process. Simplicity is the policy. The more you overcomplicate your system, the harder it will bite you in the future. Visualizing the future is a great skill to practice.
+
+**Learning**: Self-development never ends, and it needs to be consistent. Do not become a rabbit! Keep being the tortoise. This is a very cruel and changing world. Anytime you become over-confident and think of resting, you are gone. Try to read, try to assimilate that every day for at least 1 hour. Make it a habit. Stay relevant in the market.
+
+**Breadth and Depth**: Every problem in the data engineering world is unique in nature and needs special attention. The more you try to generalize it, you will lose on the resourcing and optimization. Understand what is required where. You don’t need Spark always, you don’t need SQL always. Understand all tools, but use the best one for the job.
+
+## What Should I Learn to Become a Data Engineer?
+
+As we saw above, a data engineer is a programmer, so the first thing that we need to be solid at is “Programming”. The fundamentals of programming concepts such as data modeling, OOP, basic packaging, organizing code and understanding programming paradigms is so important. Now questions can arise about what language to learn. You should learn a few basic ones (we'll talk shortly), then you can go above and beyond.
+
+### SQL & Data Modeling
+
+The most common skill that all data engineers should possess is obviously SQL, but it is the least important one from the engineering point of view. In a few years SQL will be taught in schools. This supports the largest variety of applications in the entire world for ad-hoc usage, dashboards, analytics etc. If you say you don’t know SQL then you will be in trouble. On the other hand, you have to understand data systems in depth. Understanding of data warehousing concepts and data modeling is important. SQL has no value unless you know how to model your data.
+
+### A JVM language
+
+A Java virtual machine (JVM) is a virtual machine that enables a computer to run Java programs as well as programs written in other languages that are also compiled to Java bytecode. The JVM is detailed by a specification that formally describes what is required in a JVM implementation. Let us see the advantages of JVM.
+
+- Platform independence: Java is meant for “write once, run anywhere”. Native languages like C/C++ compiled to match the specific platform, while Java code is compiled into bytecode. JVMs can interpret bytecode instructions into the native language for the specific platform.
+- Common API: the other massive benefit of using JVM is a common API to work with platform-specific resources. You don't need to worry about how to handle File Input/Output (I/O) on every possible platform when JVM will do it for you; same thing with other resources like memory, networking, input-output devices, etc. A plethora of Java libraries including common ones that are available in JRE, JVM programming language has access to all of them as if they were part of their own language.
+- Memory management: when you are on a JVM, then there is no need to reserve memory for a newly created array or to manipulate with pointers trying to simply work with arrays. The Java Virtual Machine will handle memory management. It will allocate the required memory for variables and objects. Then the memory access is tracked and the JVM subsystem Garbage Collector (GC) can reclaim unused memory when it's time. GC is not perfect and along with its usefulness it introduces slight performance degradation. The main benefit of JVM memory management is that when developing software, there is no need to think when to allocate memory and when to release it. The most popular problem with unmanaged languages like C is a memory leak that was introduced by a program itself.
+- Many languages: Java is not the only programming language that runs on the JVM. There are more advantages of using JVM than disadvantages. A lot of modern popular languages are designed specifically for the JVM, such as Scala, Groovy, Closure, and Kotlin.
+
+### The two champions - Java and Scala
+
+Java is an excellent start. Java is admired and acknowledged around the world for its versatile framework support, adoption and ease of use. Java adds value to your resume for the same reason. Once you learn Java programming basics, the OOP semantics gives you an edge compared others to rise to the next level. Learning Java is not just about the language itself, but it helps you understand the _programming paradigm_. If you are asked to create a customer object and orders object, how are you going to design their relationship? What are the parameters a customer could have, and what can you keep in common between a customer and an order? How can the quality of a customer or an order can be used as interfaces? Or a product categorized into child products? These skills are vital and once you master them, behavioral engineering design will come naturally in your solutions.
+
+Scala is the most serious investment that you could make in your lifetime. If you have ever worked with Java it will be very easy for you to pick up Scala, but even if you don’t, then also it will be quite a good friend to you. Scala has so much importance in the big data world. The best rated ETL framework (Spark) is written in Scala, the best messaging framework (Kafka) is written in Scala, and many more to name. The functional programming in Scala adds modularity in your code. Moreover, with Scala you can write your program concise. All the applications in big data today (or in the recent past) has in some way or another used Scala as backbone. Assume you create some type-safe API over Kafka Streams, no better choice than Scala. You want to tweak or extend some Spark features or want to use Datasets for creating some nice app, you need to be skilled with Scala. It is never late to invest in learning. So hurry up!
+
+### Hadoop Basics & Distributed Computing
+
+Hadoop is just like learning the alphabet! Hadoop brought life into big data. It is old, but it is gold. You might think, "this is 2022, why is Ani emphasizing learning Hadoop?!" Well, there are a lot of people who think we can avoid it. The truth is, without knowing a true sense distributed system which supports scalable storage and compute services, one can not become a data engineer in reality. Learn Hadoop basics with some serious dedication, learn how it stores files, how it retrieves them. How the partitioning works, how even it looks up to a file chunk so efficiently, what is the best way to store the metadata and how to govern it. Once you learn Hadoop in depth the rest of the path becomes very easy and interesting.
+
+### Spark
+
+Today in the big data world there is no escape from Spark. Spark is everywhere. You want to read data from Amazon S3 and load it to Cassandra at scale, use Spark. You want to read from Apache Kafka and run some transformation on it and store it to Mongo DB, use Spark. You want to create a streaming app on top of message queues, you use Spark there too. The cheapest, the smartest distributed computing framework ever created. The most important part of Spark that we have to learn is [Spark](https://rockthejvm.com/p/spark) core, Spark SQL, [Structured Streaming](https://rockthejvm.com/p/spark-streaming), [Spark Optimization](https://rockthejvm.com/p/spark-optimization). The knowledge of tuning a Spark cluster and resources push you to become the best rated data engineer.
+
+### Kafka
+
+There are and will be a lot of streaming/messaging systems but Kafka is just the best in the lot, in a similar place as Spark in computing engines. You can learn any distributed message queues and understand others. Considering the community support and live applications in the entire world no one can beat Kafka at this moment. Bag it!
+
+### Scripting
+
+A scripting language like Unix shell or Python will be good enough to get basic operations such as file handling, calling APIs etc. Suppose you want to run a Spark job on AWS EMR with a spark-submit. You can do it by a Python shell with boto3 libraries. Scripts can achieve many things: avoid repetitive work by automations, backups, monitoring, "firefighting" in case of a disaster, and more.
+
+## Non-Technical Skills of Data Engineers
+
+We have enough to read about the technical acumen you need to become a data engineer. Now there are few other important qualities to become a “Successful” engineer. Yeah, it needs a lot of behavioral qualities as well.
+
+Do we have enough of them?
+
+Even possessing a plethora of technical skills, these days a data engineer in true sense is just like witnessing a unicorn outside your courtyard. Let’s talk about some key non-technical skill sets a data engineer should possess before even thinking about a career shift.
+
+- Have some “peace of mind”: Emerging professionals around the world, be it software engineers or data engineers, are restless and running after quick success and collecting fame with shortcuts. There’s no shortcut to success. Peace of mind is such a nice quality to grow within, which can help anyone to be firm in critical situations and keep the path steady.
+- Be a listener first: Most of the time we make mistakes just because we do not wait to understand the problem holistically. We don’t listen but we propose opinions. 50% of the job is done when we listen to the problem statement and then think.
+- Take it slow and steady: Being first/fast doesn't make you optimal. Look into your solution 5 times more to validate with a 360 degree view. You miss it, and someone will have to correct you, which wastes everyone's time and your growth will suffer.
+- Read regularly: Reading “anything” of your interest at least 1 hour every day will give you such value that you can not even imagine. Take some idea that sparked your interest, read about it, live it and dream about it.
+- Remember you don’t know everything: Be a humble student, that makes 100% of the job. You are not an authority of anything but your character, your true self.
+- Ask for help: Ask for help from colleagues and professionals, no question is a bad question. Often, a question for one is a question for all, and everybody can learn.
+
+## Conclusion
+
+_This is Daniel again — I hope this article gave you some insight and inspiration of what a Data Engineer means, what kind of skills you need to be a successful one like Ani, and what you should learn and focus on. Ani writes on his own [Medium blog](https://thedatafreak.medium.com) and you can find him on [LinkedIn](https://www.linkedin.com/in/anigos/) and in the Rock the JVM private room on Slack, where he helps everyone interested in Data Engineering._
diff --git a/_posts/2022-06-10-pulsar-flink.md b/_posts/2022-06-10-pulsar-flink.md
new file mode 100644
index 000000000000..e96988c2fff5
--- /dev/null
+++ b/_posts/2022-06-10-pulsar-flink.md
@@ -0,0 +1,861 @@
+---
+title: "Stateful Streams with Apache Pulsar and Apache Flink"
+date: 2022-06-14
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [pulsar, flink, streaming]
+excerpt: "We discuss how you can integrate Apache Pulsar\ with Apache Flink to perform data enrichment with state from different topics."
+---
+
+_This article is a collaboration between myself (Daniel) and [Giannis Polyzos](https://www.linkedin.com/in/polyzos/), one of the earliest students of Rock the JVM back in 2017. Giannis is now a Senior Engineer and a contributor to Apache Pulsar, a promising new toolkit for distributed messaging and streaming. Last time we integrated [Pulsar with Spark](/pulsar-spark); in this post we combine Apache Pulsar and Apache Flink for powerful data stream processing._
+
+
+
+Before we start, some basic familiarity with Apache Pulsar and Apache Flink is required — Rock the JVM has an [Apache Flink course](https://rockthejvm.com/p/flink) if you're interested. To better understand the implementation in this blog post, we suggest getting familiar with the basic concepts of Apache Pulsar and Apache Flink. See the [Additional Resources](#7-additional-resources) section.
+
+The code in this article is in **Java**, although it can be adapted to Scala as well.
+For the best experience following this article, we recommend referring to [this repository](https://github.com/polyzos/pulsar-flink-stateful-streams) while replicating the project locally in your own dev environment. You can also clone the repository and work directly on the code there.
+
+### 1. Introduction
+
+Typical Streaming data architectures include a streaming storage layer like Apache Pulsar that serves as the backbone of the infrastructure.
+Stateful stream processing is also required to deliver advanced analytics for your users; a stream computing engine
+like Apache Flink is required to handle time-based event computations, especially when they require managing large state.
+Data often resides inside multiple different topics in a streaming storage layer, and it's important to be able to combine data from _multiple input topics_.
+
+In this blog post, we will walk through how you can use Apache Flink to _enrich real time data streams_ with data that resides into large changelog topics.
+We will use Apache Pulsar as our streaming storage layer.
+Apache Pulsar and Apache Flink have a strong integration together and enable a Unified Batch and Streaming Architecture.
+If you are interested about this type of architecture, [this video](https://www.youtube.com/watch?v=2MpiE238Pzw) can be helpful.
+
+## 2. The Example: Data From an Online Store
+
+
+
+Our example use case is an online store and users come online to place orders for different items. Our fundamental types involved are users, items (products) and orders.
+Every new order is written into an `orders` topic, and similarly for users or items - written into the `users` and `items` topics respectively.
+We treat `users` and `items` topics as _changelog_ streams — this means that the events written in those topics will be
+a key-value pair and for each unique key we are only interested in the _latest_ value.
+
+For example, if `user1` updates the phone number we are ony interested in the latest updated value (the new phone number). The same goes for a product.
+We will consider these changelog topics as our _state_.
+
+A common use case in streaming systems is combining data from different topics, in order to perform some kind of data validation or data enrichment.
+In our example, we can enrich the input `order` events with the _state_, for example to query the user and product information to be
+able to take actions such as
+
+- sending out an email thanking our user for their purchase
+- calculating some reward points to see if they are eligible for a discount coupon
+- recommending something similar to the product they just bought
+
+Our focus for this article is enriching an input event stream with information from events in other topics.
+
+We will take a hands-on approach and better understand how we can:
+1. Connect Apache Pulsar with Flink and verify we can consume events from different topics
+2. Use Flink's `ProcessFunction`s to perform data enrichment
+3. Use _side outputs_ to account for scenarios where state is not present, and we want to further investigate why
+4. Use RocksDB for large state we can not keep in memory
+5. Recover from failures with _checkpoints_ and _restart strategies_
+6. How we can resume our Flink Job using _savepoints_
+
+There is a lot to cover here, so we will build on them incrementally. Let's jump right into it.
+
+## 2. Pre-Flight Check
+
+Before we start with the implementation let's make sure we have our dependencies in place and our environment setup.
+The examples here will be in Java, so let's add the following dependencies to our `pom.xml` file.
+
+```xml
+
+ 11
+ 11
+
+ 1.14.3
+ 2.12
+
+
+
+
+ org.apache.flink
+ flink-clients_${scala.version}
+ ${flink.version}
+
+
+
+ org.apache.flink
+ flink-streaming-java_${scala.version}
+ ${flink.version}
+
+
+
+ org.apache.flink
+ flink-connector-pulsar_${scala.version}
+ ${flink.version}
+
+
+
+ org.apache.flink
+ flink-statebackend-rocksdb_${scala.version}
+ ${flink.version}
+
+
+
+ org.projectlombok
+ lombok
+ 1.18.22
+ provided
+
+
+
+ ch.qos.logback
+ logback-classic
+ 1.2.11
+
+
+```
+
+We also require an instance of Pulsar and Flink, so we will use [docker-compose](https://docs.docker.com/compose/) to create them as local services.
+Create a `docker-compose.yml` file and add the following:
+
+```shell
+version: '3.7'
+services:
+ jobmanager:
+ image: flink:1.14.4-scala_2.12-java11
+ ports:
+ - "8081:8081"
+ command: jobmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: jobmanager
+
+ taskmanager:
+ image: flink:1.14.4-scala_2.12-java11
+ depends_on:
+ - jobmanager
+ command: taskmanager
+ scale: 1
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: jobmanager
+ taskmanager.numberOfTaskSlots: 2
+
+ pulsar:
+ image: apachepulsar/pulsar:2.9.1
+ container_name: pulsar
+ ports:
+ - "8080:8080"
+ - "6650:6650"
+ environment:
+ PULSAR_MEM: " -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g"
+ systemTopicEnabled: "true"
+ topicLevelPoliciesEnabled: "true"
+ transactionCoordinatorEnabled: "true"
+ command: >
+ /bin/bash -c
+ " bin/apply-config-from-env.py conf/standalone.conf
+ && bin/pulsar standalone"
+```
+
+There are a few things to note here:
+- First, we create a Flink Cluster with one Task Manager which has 2 slots, and a Pulsar cluster.
+- We expose ports 8080 for Pulsar, as well as port 6650 which is the Pulsar broker port.
+- We also enable some configurations: `systemTopicEnabled` and `topicLevelPoliciesEnabled` allow us to use topic-level policies for infinite data retention.
+- Then we have `transactionCoordinatorEnabled` which enables the transaction coordinator which is used by the Pulsar-Flink connector.
+
+So let's start our clusters by running **docker-compose up**.
+
+With our clusters up and running we need to create our topics and apply some topic policies.
+Retention topic policies will also allow us to reply our events in a case of failure if required.
+We can run this [script](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/setup.sh) to perform this setup. For reference, please see it below as well.
+It will create 3 partitioned topics (orders, users and items) each with one partition and set an infinite retention policy for the changelog topics.
+
+```shell
+docker exec -it pulsar bin/pulsar-admin topics create-partitioned-topic -p 1 persistent://public/default/orders
+docker exec -it pulsar bin/pulsar-admin topics create-partitioned-topic -p 1 persistent://public/default/users
+docker exec -it pulsar bin/pulsar-admin topics create-partitioned-topic -p 1 persistent://public/default/items
+
+docker exec -it pulsar bin/pulsar-admin topics list public/default
+
+docker exec -it pulsar bin/pulsar-admin topics set-retention -s -1 -t -1 persistent://public/default/users
+docker exec -it pulsar bin/pulsar-admin topics set-retention -s -1 -t -1 persistent://public/default/items
+
+docker exec -it pulsar bin/pulsar-admin topics get-retention persistent://public/default/users
+docker exec -it pulsar bin/pulsar-admin topics get-retention persistent://public/default/items
+
+```
+
+**Note:** We use partitioned topics in order to be able to increase the number of partitions later, in case we need to scale. Otherwise, if we had an unpartitioned topic we would have to create a new partitioned topic and transfer all the data to this new topic.
+
+## 3. Reading Data From Pulsar
+
+With our setup in place, let's see some data-reading code. The data model can be found [here](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/models) if you want to grab the same data definitions for our fictitious online store. After that, we recommend taking a look and maybe grabbing [these utility functions](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/utils) as well as the constants [here](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/config) as we're going to focus more on the Flink-Pulsar integration. The fictitious data we're going to push to Pulsar was generated and can be found in [these files](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/data).
+
+First, we will run the producers to simulate some users and items created in our system.
+You can find the producer code [here](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/producers/LookupDataProducer.java). For reference, please find it below as well:
+
+```java
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
+import java.util.stream.Stream;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.impl.schema.JSONSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LookupDataProducer {
+ private static final Logger logger
+ = LoggerFactory.getLogger(LookupDataProducer.class);
+
+ public static void main(String[] args) throws IOException {
+ Stream userStream = DataSourceUtils.loadDataFile(AppConfig.USERS_FILE_PATH)
+ .map(DataSourceUtils::lineAsUser);
+
+ Stream- itemsStream = DataSourceUtils.loadDataFile(AppConfig.ITEMS_FILE_PATH)
+ .map(DataSourceUtils::lineAsItem);
+
+ logger.info("Creating Pulsar Client ...");
+ PulsarClient pulsarClient = ClientUtils.initPulsarClient(AppConfig.token);
+
+ logger.info("Creating User Producer ...");
+ Producer
userProducer
+ = createProducer(pulsarClient, "user-producer", AppConfig.USERS_TOPIC, User.class);
+
+ AtomicInteger userCounter = new AtomicInteger();
+ for (Iterator it = userStream.iterator(); it.hasNext(); ) {
+ User user = it.next();
+
+ produceMessage(userProducer, String.valueOf(user.getId()), user, userCounter);
+ }
+
+ Producer- itemProducer
+ = createProducer(pulsarClient, "item-producer", AppConfig.ITEMS_TOPIC, Item.class);
+
+ AtomicInteger itemsCounter = new AtomicInteger();
+ for (Iterator
- it = itemsStream.iterator(); it.hasNext(); ) {
+ Item item = it.next();
+
+ produceMessage(itemProducer, String.valueOf(item.getId()), item, itemsCounter);
+ }
+
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ logger.info("Sent '{}' user records and '{}' item records.", userCounter.get(), itemsCounter.get());
+ logger.info("Closing Resources...");
+ try {
+ userProducer.close();
+ itemProducer.close();
+ pulsarClient.close();
+ } catch (PulsarClientException e) {
+ e.printStackTrace();
+ }
+ }));
+ }
+
+ private static
Producer createProducer(PulsarClient pulsarClient,
+ String producerName,
+ String topicName,
+ Class classz) throws PulsarClientException {
+ logger.info("Creating {} Producer ...", classz.getSimpleName());
+ return pulsarClient.newProducer(JSONSchema.of(classz))
+ .producerName(producerName)
+ .topic(topicName)
+ .blockIfQueueFull(true)
+ .create();
+ }
+
+ private static void produceMessage(Producer producer, String key, T value, AtomicInteger counter) {
+ producer.newMessage()
+ .key(key)
+ .value(value)
+ .eventTime(System.currentTimeMillis())
+ .sendAsync()
+ .whenComplete(callback(counter));
+ }
+
+ private static BiConsumer callback(AtomicInteger counter) {
+ return (id, exception) -> {
+ if (exception != null) {
+ logger.error("❌ Failed message: {}", exception.getMessage());
+ } else {
+ logger.info("✅ Acked message {} - Total {}", id, counter.getAndIncrement());
+ }
+ };
+ }
+}
+```
+
+This data producer's main function does the following:
+
+- loads the data files with one of the utilities earlier — nothing more than mere CSV parsing
+- creates a Pulsar client
+- creates data producers for users and for items
+- pushes each user and each item to Pulsar one by one
+- keeps track of the total records with an `AtomicInteger` (we're doing parallel processing, remember?)
+- closes the resources cleanly
+
+After running the producer we should see an output similar to the following:
+
+```shell
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:26 - Total 2190
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:27 - Total 2191
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:28 - Total 2192
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:29 - Total 2193
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:30 - Total 2194
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:31 - Total 2195
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:32 - Total 2196
+17:35:43.861 INFO [pulsar-client-io-1-1] io.ipolyzos.producers.LookupDataProducer - ✅ Acked message 35:98:0:33 - Total 2197
+```
+
+The next step is to create 3 **Pulsar-Flink** sources that will listen and consume events in the topics.
+We will use the Pulsar Flink connector. You can find more information about the connectors [here](https://nightlies.apache.org/flink/flink-docs-release-1.14/docs/connectors/datastream/pulsar/)
+The following code snippet shows a basic Pulsar-Flink source:
+
+```shell
+PulsarSource pulsarSource = PulsarSource.builder()
+ .setServiceUrl("pulsar://localhost:6650")
+ .setAdminUrl("http://localhost:8080")
+ .setStartCursor(StartCursor.earliest())
+ .setTopics("my-topic")
+ .setDeserializationSchema(PulsarDeserializationSchema.flinkSchema(new SimpleStringSchema()))
+ .setSubscriptionName("my-subscription")
+ .setSubscriptionType(SubscriptionType.Exclusive)
+ .build();
+```
+
+There are a few things to highlight here:
+
+1. We need to provide the service and admin URLs, the same ones we exposed on our docker-compose file.
+2. `StartCursor` is the position we want to start consuming messages from, and `earliest()` indicates we want to start consuming messages from the beginning of the topic.
+3. We must pass the topic name, the name of the subscription and also the _type_ of the subscription. You can find more information around Pulsar Subscription Types [here](https://pulsar.apache.org/docs/2.3.2/concepts-messaging/#subscription-types)
+4. Finally, we need to specify what type of schema we wish to consume. In the example above it's of type String, but we will use `JsonSchema` for our application.
+
+The following code snippet shows how to connect to our topics, but you can find the complete code [here](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/compute/v1/EnrichmentStream.java)
+```java
+// import the data model as well
+import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.connector.pulsar.source.PulsarSource;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.pulsar.client.api.SubscriptionType;
+
+import java.time.Duration;
+
+public class EnrichmentStream {
+ public static void main(String[] args) throws Exception {
+ // 1. Initialize the execution environment
+ StreamExecutionEnvironment env = EnvironmentUtils.initEnvWithWebUI(false);
+
+ // 2. Initialize Sources
+ PulsarSource userSource =
+ EnvironmentUtils.initPulsarSource(
+ AppConfig.USERS_TOPIC,
+ "flink-user-consumer",
+ SubscriptionType.Exclusive,
+ StartCursor.earliest(),
+ User.class);
+
+ PulsarSource- itemSource =
+ EnvironmentUtils.initPulsarSource(
+ AppConfig.ITEMS_TOPIC,
+ "flink-items-consumer",
+ SubscriptionType.Exclusive,
+ StartCursor.earliest(),
+ Item.class);
+
+ PulsarSource
orderSource =
+ EnvironmentUtils.initPulsarSource(
+ AppConfig.ORDERS_TOPIC,
+ "flink-orders-consumer",
+ SubscriptionType.Exclusive,
+ StartCursor.earliest(),
+ Order.class);
+
+ WatermarkStrategy watermarkStrategy =
+ WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofSeconds(5))
+ .withTimestampAssigner(
+ (SerializableTimestampAssigner) (order, l) -> order.getCreatedAt()
+ );
+
+ // 3. Initialize Streams
+ DataStream userStream =
+ env.fromSource(userSource, WatermarkStrategy.noWatermarks(), "Pulsar User Source")
+ .name("pulsarUserSource")
+ .uid("pulsarUserSource");
+
+ DataStream- itemStream =
+ env.fromSource(itemSource, WatermarkStrategy.noWatermarks(), "Pulsar Items Source")
+ .name("pulsarItemSource")
+ .uid("pulsarItemSource");
+
+ DataStream
orderStream = env.fromSource(orderSource, watermarkStrategy, "Pulsar Orders Source")
+ .name("pulsarOrderSource")
+ .uid("pulsarOrderSource");
+
+ orderStream
+ .print()
+ .uid("print")
+ .name("print");
+ env.execute("Order Enrichment Stream");
+ }
+}
+
+```
+
+We also need to create a `WatermarkStrategy` for our orders input data stream to handle late order events.
+Event Time with be tracked by the creation time within the `Order` event.
+
+If you need to learn about watermarks or event time/processing time, Rock the JVM's [Apache Flink course](https://rockthejvm.com/p/flink) goes into great detail, as they are critical topics for stateful distributed data processing.
+
+Note that while the users and items streams already have data written to Pulsar, the orders stream does not - we will trigger the data flow in Flink by sending orders data to Pulsar from a different process (e.g. in the IDE's local run command) after we start the Flink application.
+
+The last step is to actually package and deploy our code on our cluster.
+To make it easier we can use the helper script [here](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/deploy.sh), which you can also find below:
+
+```shell
+mvn clean package
+
+docker cp \
+ target/pulsar-flink-stateful-streams-0.1.0.jar \
+ pulsar-flink-stateful-streams_taskmanager_1:opt/flink/job.jar
+
+docker exec -it pulsar-flink-stateful-streams_taskmanager_1 ./bin/flink run \
+ --class io.ipolyzos.compute.v1.EnrichmentStream \
+ job.jar
+```
+
+Run `./deploy.sh` and navigate to the terminal we ran the `docker-compose up` command.
+Give it a few seconds and then the job should be deployed. After that, we can run the `OrdersDataSource` found [here](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/producers/OrdersDataSource.java). The code can also be found below:
+
+```java
+// also include the data model
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.impl.schema.JSONSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OrdersDataSource {
+ private static final Logger logger
+ = LoggerFactory.getLogger(OrdersDataSource.class);
+ public static void main(String[] args) throws IOException, InterruptedException {
+ Stream sourceStream = DataSourceUtils.loadDataFile(AppConfig.ORDERS_FILE_PATH)
+ .map(DataSourceUtils::lineAsOrder);
+
+ logger.info("Creating Pulsar Client ...");
+ PulsarClient pulsarClient = ClientUtils.initPulsarClient(AppConfig.token);
+
+ logger.info("Creating Orders Producer ...");
+ Producer ordersProducer
+ = pulsarClient.newProducer(JSONSchema.of(Order.class))
+ .producerName("order-producers")
+ .topic(AppConfig.ORDERS_TOPIC)
+ .create();
+
+ AtomicInteger counter = new AtomicInteger(1);
+ for (Iterator it = sourceStream.iterator(); it.hasNext(); ) {
+ Order order = it.next();
+
+ ordersProducer.newMessage()
+ .value(order)
+ .eventTime(System.currentTimeMillis())
+ .send();
+
+ logger.info("✅ Total {} - Sent: {}", counter.getAndIncrement(), order);
+ }
+
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ logger.info("Sent '{}' orders.", counter.get());
+ logger.info("Closing Resources...");
+ try {
+ ordersProducer.close();
+ pulsarClient.close();
+ } catch (PulsarClientException e) {
+ e.printStackTrace();
+ }
+ }));
+ }
+}
+```
+
+After the Flink application is running, we can trigger the application above, and we should see logs similar to this (in the docker-compose terminal):
+
+```shell
+taskmanager_1 | Order(invoiceId=44172, lineItemId=192933, userId=145493, itemId=602, itemName=prize-winning instrument, itemCategory=instrument, price=48.40000000000001, createdAt=1469834917000, paidAt=1469700797000)
+taskmanager_1 | Order(invoiceId=44172, lineItemId=385101, userId=145493, itemId=3362, itemName=matte instrument, itemCategory=instrument, price=55.0, createdAt=1469834917000, paidAt=1469700797000)
+taskmanager_1 | Order(invoiceId=44172, lineItemId=285219, userId=145493, itemId=2584, itemName=industrial-strength instrument, itemCategory=instrument, price=132.0, createdAt=1469834917000, paidAt=1469700797000)
+taskmanager_1 | Order(invoiceId=245615, lineItemId=229127, userId=112915, itemId=1982, itemName=extra-strength gadget wrapper, itemCategory=gadget, price=71.2, createdAt=1463179333000, paidAt=1463195221000)
+taskmanager_1 | Order(invoiceId=245615, lineItemId=384894, userId=112915, itemId=564, itemName=gadget wrapper, itemCategory=gadget, price=35.6, createdAt=1463179333000, paidAt=1463195221000)
+taskmanager_1 | Order(invoiceId=343211, lineItemId=258609, userId=34502, itemId=1257, itemName=tool warmer, itemCategory=tool, price=27.500000000000004, createdAt=1421975214000, paidAt=1422101333000)
+```
+
+Congrats! We have successfully consumed messages from Pulsar.
+
+## 4. Performing Data Enrichment in Flink
+
+We verified we can read our events. The next step is data enrichment, i.e "query" user and item information from the _changelog_ topics.
+To do this, we'll use the `connect` the orders stream with the user stream and (separately) with the items stream. We talk about the `connect` function in great detail in the Flink course — in short, it's similar to a "join" where the records with the same key are combined into a tuple. Each tuple is then subject to a `ProcessFunction` (the core Flink abstraction) so that we can adjust the data to a format of our choosing.
+
+Aside from the earlier `EnrichmentStream` class, we now add the following in [version 2](https://github.com/polyzos/pulsar-flink-stateful-streams/blob/main/src/main/java/io/ipolyzos/compute/v2/EnrichmentStream.java):
+```java
+DataStream orderWithUserDataStream = orderStream
+ .keyBy(Order::getUserId)
+ .connect(userStream.keyBy(User::getId))
+ .process(new UserLookupHandler())
+ .uid("usersLookup")
+ .name("usersLookup");
+
+SingleOutputStreamOperator enrichedOrderStream = orderWithUserDataStream
+ .keyBy(OrderWithUserData::getItemId)
+ .connect(itemStream.keyBy(Item::getId))
+ .process(new ItemLookupHandler())
+ .uid("itemsLookup")
+ .name("itemsLookup");
+```
+
+The process function implementation looks like this:
+
+```java
+
+// import the data model
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class UserLookupHandler extends CoProcessFunction {
+ private static final Logger logger = LoggerFactory.getLogger(UserLookupHandler.class);
+ private ValueState userState;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ logger.info("{}, initializing state ...", this.getClass().getSimpleName());
+
+ userState = getRuntimeContext()
+ .getState(
+ new ValueStateDescriptor("userState", User.class)
+ );
+ }
+
+ @Override
+ public void processElement1(Order order, CoProcessFunction.Context context,
+ Collector collector) throws Exception {
+ User user = userState.value();
+ if (user == null) {
+ logger.warn("Failed to find state for id '{}'", order.getUserId());
+ } else {
+ collector.collect(order.withUserData(user));
+ }
+ }
+
+ @Override
+ public void processElement2(User user,
+ CoProcessFunction.Context context,
+ Collector collector) throws Exception {
+ userState.update(user);
+ }
+}
+```
+
+We extend the `CoProcessFunction` that processes elements of two input streams (here users and orders) and produces a single output (here `OrderWithUserData`).
+The function will be called for every event coming from each input streams, and can produce zero or more output elements.
+Note that for each user record we receive we use the `Value` state in order to store it.
+Then, for every incoming order to try and "query" this state, and if there is a matching key, we enrich the order event.
+(we'll handle missing state later).
+The implementation for enriching with `Item` values is similar:
+
+```java
+public class ItemLookupHandler extends CoProcessFunction {
+ private static final Logger logger = LoggerFactory.getLogger(UserLookupHandler.class);
+ private ValueState- itemState;
+
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ logger.info("{}, initializing state ...", this.getClass().getSimpleName());
+
+ itemState = getRuntimeContext()
+ .getState(
+ new ValueStateDescriptor
- ("itemState", Item.class)
+ );
+ }
+
+ @Override
+ public void processElement1(OrderWithUserData order,
+ CoProcessFunction
.Context context,
+ Collector collector) throws Exception {
+ Item item = itemState.value();
+ if (item == null) {
+ logger.warn("Failed to find state for id '{}'", order.getItemId());
+ } else {
+ collector.collect(
+ new EnrichedOrder(
+ order.getInvoiceId(),
+ order.getLineItemId(),
+ order.getUser(),
+ item,
+ order.getCreatedAt(),
+ order.getPaidAt()
+ )
+ );
+ }
+ }
+
+ @Override
+ public void processElement2(Item item,
+ CoProcessFunction.Context context,
+ Collector collector) throws Exception {
+ itemState.update(item);
+ }
+}
+```
+
+You can find a full implementation under the **v2** package [here](https://github.com/polyzos/pulsar-flink-stateful-streams/tree/main/src/main/java/io/ipolyzos/compute/v2).
+We can now package and redeploy our application and verify it works.
+Make sure to modify the `deploy.sh` script to point to the updated **v2** version file:
+
+```shell
+mvn clean package
+
+docker cp \
+ target/pulsar-flink-stateful-streams-0.1.0.jar \
+ pulsar-flink-stateful-streams_taskmanager_1:opt/flink/job.jar
+
+docker exec -it pulsar-flink-stateful-streams_taskmanager_1 ./bin/flink run \
+ --class io.ipolyzos.compute.v2.EnrichmentStream \
+ job.jar
+```
+
+Following our previous steps:
+1. Run the `deploy.sh` script
+2. Generate some `Order` events
+3. Check the logs
+
+We should see an output similar to this:
+```shell
+taskmanager_1 | EnrichedOrder(invoiceId=67052, lineItemId=326416, user=User(id=88300, firstName=Davis, lastName=MDavis1997@earthlink.edu, emailAddress=MDavis1997@earthlink.edu, createdAt=1441790913000, deletedAt=-1, mergedAt=-1, parentUserId=-1), item=Item(id=930, createdAt=1388876010000, adjective=, category=module, modifier=, name=module, price=100.0), createdAt=1443643093000, paidAt=1443745976000)
+taskmanager_1 | EnrichedOrder(invoiceId=67052, lineItemId=146888, user=User(id=88300, firstName=Davis, lastName=MDavis1997@earthlink.edu, emailAddress=MDavis1997@earthlink.edu, createdAt=1441790913000, deletedAt=-1, mergedAt=-1, parentUserId=-1), item=Item(id=80, createdAt=1372810111000, adjective=rechargable, category=module, modifier=cleaner, name=rechargable module cleaner, price=78.0), createdAt=1443643093000, paidAt=1443745976000)
+taskmanager_1 | EnrichedOrder(invoiceId=67052, lineItemId=204597, user=User(id=88300, firstName=Davis, lastName=MDavis1997@earthlink.edu, emailAddress=MDavis1997@earthlink.edu, createdAt=1441790913000, deletedAt=-1, mergedAt=-1, parentUserId=-1), item=Item(id=336, createdAt=1385261877000, adjective=fuzzy, category=module, modifier=, name=fuzzy module, price=100.0), createdAt=1443643093000, paidAt=1443745976000)
+taskmanager_1 | EnrichedOrder(invoiceId=220846, lineItemId=48384, user=User(id=182477, firstName=Powell, lastName=MarinaPowell@mail.com, emailAddress=MarinaPowell@mail.com, createdAt=1485101903000, deletedAt=-1, mergedAt=-1, parentUserId=-1), item=Item(id=1514, createdAt=1387918171000, adjective=miniature, category=apparatus, modifier=cleaner, name=miniature apparatus cleaner, price=99.0), createdAt=1493699951000, paidAt=1493632923000)
+taskmanager_1 | EnrichedOrder(invoiceId=220846, lineItemId=230208, user=User(id=182477, firstName=Powell, lastName=MarinaPowell@mail.com, emailAddress=MarinaPowell@mail.com, createdAt=1485101903000, deletedAt=-1, mergedAt=-1, parentUserId=-1), item=Item(id=2425, createdAt=1372279813000, adjective=, category=apparatus, modifier=, name=apparatus, price=300.0), createdAt=1493699951000, paidAt=1493632923000)
+taskmanager_1 | EnrichedOrder(invoiceId=278358, lineItemId=129026, user=User(id=97081, firstName=Adebayo, lastName=SunitaAdebayo@inbox.info, emailAddress=SunitaAdebayo@inbox.info, createdAt=1446040475000, deletedAt=-1, mergedAt=-1, parentUserId=-1), item=Item(id=3435, createdAt=1373472723000, adjective=industrial-strength, category=widget, modifier=cleaner, name=industrial-strength widget cleaner, price=5.4), createdAt=1453951447000, paidAt=1454087954000)
+```
+
+We have successfully enriched our `Order`s with `User` and `Item` information.
+
+At this point there are two questions we need to address:
+1. How can we investigate records that have no matching user and/or item record id?
+2. Since our state is kept in memory, how can we handle state that is too large to fit in memory?
+
+Let's see how can achieve this.
+
+## 5. Using Side Outputs for Missing State
+
+Working with distributed systems, we want to be able to handle the "unhappy paths", i.e. an unexpected behavior within our system.
+When an order is submitted we assume the information for the user and a purchased item are always present, but can we guarantee this is always the case?
+
+In order to have more visibility we introduce Flink's [Side Outputs](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/datastream/side_output/).
+You can think of Side Outputs like a branch of a stream that you can use to redirect events that don't comply with your expected behavior and need to be propagated to a different output downstream, like printing them to the console, pushing them to another Pulsar topic or storing into a database.
+
+With this approach, if we hit a scenario that a user or item event is missing, we can propagate the order event downstream.
+We might not be sure why this happened, but at least we have visibility that it happened and we can investigate more later.
+
+In order to use side outputs, we need to modify our `ProcessFunction` logic:
+```java
+public class UserLookupHandler extends CoProcessFunction {
+ private static final Logger logger = LoggerFactory.getLogger(UserLookupHandler.class);
+ private final OutputTag missingStateTag; // (1)
+ private ValueState userState;
+
+ public UserLookupHandler(OutputTag missingStateTag) {
+ this.missingStateTag = missingStateTag;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ logger.info("{}, initializing state ...", this.getClass().getSimpleName());
+
+ userState = getRuntimeContext()
+ .getState(
+ new ValueStateDescriptor("userState", User.class)
+ );
+ }
+
+ @Override
+ public void processElement1(Order order, CoProcessFunction.Context context,
+ Collector collector) throws Exception {
+ User user = userState.value();
+ if (user == null) {
+ logger.warn("Failed to find state for id '{}'", order.getUserId());
+ EnrichedOrder enrichedOrder =
+ new EnrichedOrder(order.getInvoiceId(), order.getLineItemId(), null, null, order.getCreatedAt(),
+ order.getPaidAt());
+ context.output(missingStateTag, enrichedOrder); // (2)
+ } else {
+ collector.collect(order.withUserData(user));
+ }
+ }
+
+ @Override
+ public void processElement2(User user,
+ CoProcessFunction.Context context,
+ Collector collector) throws Exception {
+ userState.update(user);
+ }
+}
+```
+
+Note the `(1)` and `(2)` marks in the code:
+
+- **(1)** We create an OutputTag typed with our output event `OrderWithUserData`.
+- **(2)** If a key is not present in our state for a particular ID, then we add the event to the side output.
+
+We also need to modify our main `EnrichmentStream` class to support Side Outputs:
+```java
+ final OutputTag missingStateTagUsers = new OutputTag<>("missingState#User"){};
+ final OutputTag missingStateTagItems = new OutputTag<>("missingState#Item"){};
+
+ enrichedOrderStream.getSideOutput(missingStateTagUsers)
+ .printToErr()
+ .name("MissingUserStateSink")
+ .uid("MissingUserStateSink");
+
+ enrichedOrderStream.getSideOutput(missingStateTagItems)
+ .printToErr()
+ .name("MissingItemStateSink")
+ .uid("MissingItemStateSink");
+```
+
+Here we create two side outputs - one for missing user events and one for item events. Then we extract the side outputs from our output stream and print it.
+
+**Note:** It's also worth highlighting the use of `name` and `uid` for each operator.
+Specifying names for your operators can be considered as _best practice_ for your Flink Job.
+This is useful to more easily identify the operators in the Flink UI. This also comes in handy when you need to use _savepoints_ to resume your job, after a code modification or scaling requirement (more on that later).
+
+You can find the full implementation under the **v3** package [here](https://github.com/polyzos/pulsar-flink-stateful-streams/tree/main/src/main/java/io/ipolyzos/compute/v3)
+
+We can also run the application in the same style:
+
+- rebuild and redeploy with `deploy.sh`, making sure to point to the `v3` implementation
+- run the `EnrichmentStream` (the v3 of it as well)
+- look at the outputs
+
+We have covered a lot so far. So let's take a moment and walk through the implementation and what we have achieved so far.
+
+As a quick recap:
+1. We have created 3 input sources that consume data from 3 different Pulsar topics.
+2. The `orders` topic is a real-time event stream. `users` and `orders` topics are _changelog_ streams (i.e maintain the last state per key).
+3. We leverage Flink's process function along with Flink's state to enrich the input `orders` event stream with data from `user` and `item` events.
+4. We introduced Side Outputs to handle events with no matching keys in the `user` or `items` state.
+In a real life scenario you can't email a user without their email information or before you have verified they have given consent, right?
+
+We are left with one open question: how we can provide _fault-tolerance_ guarantees for our streaming job.
+We want to account for scenarios that our state grows quite large to fit in memory and/or our job crashes, and we need to recover fast.
+
+## 6. Making our job Fault Tolerant
+
+> Checkpoints make state in Flink Fault Tolerant by allowing state and the corresponding stream positions to be recovered, thereby giving the application the same semantics as a failure-free execution.
+
+We can easily enable checkpoints by applying some configuration option. We will enable the required configuration option and add a `RestartStrategy` to let Flink try and restart a job upon an Exception.
+Combining a `RestartStrategy` with Checkpoints gives our job the ability to recover in case of an Exception, for example due to some temporary connection error.
+For critical exceptions that will continually crash our job, the best approach will be to actually kill the job, apply a fix and recover with a [savepoint](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/state/savepoints/)
+
+Add the following configurations in the `EnrichmentStream` application:
+
+```java
+ // Checkpoints configurations
+ env.enableCheckpointing(5000);
+ env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500);
+ env.getCheckpointConfig().setCheckpointStorage(AppConfig.checkpointDir);
+ env.setStateBackend(new EmbeddedRocksDBStateBackend());
+ env.getCheckpointConfig().setExternalizedCheckpointCleanup(
+ CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION
+ );
+
+ // Configure Restart Strategy
+ env.setRestartStrategy(
+ RestartStrategies.fixedDelayRestart(5, Time.of(10, TimeUnit.SECONDS))
+ );
+```
+
+Deploy again and navigate to the flink UI, if you go to the Job UI you should see the following and checkpoints being created.
+
+
+
+
+
+
+Now let's open a new terminal and run the following command to connect to our `taskmanager` container
+
+```shell
+docker exec -it pulsar-flink-stateful-streams_taskmanager_1 bash
+```
+
+
+
+Under the `/opt/flink` directory you should see a directory named `checkpoints`
+
+```shell
+# the actual id might differ
+cd checkpoints/0c83cf0320b3fc6fdcdb3d8323c27503/
+```
+
+This is the directory flink uses to store all the checkpoints.
+This means that when we get a savepoint, our job crashes, or we stop it and need to restore it from a particular checkpoint. We can do so by using this checkpoint directory.
+
+So let's test this out - by killing our running flink job and then try to restart, using the following command:
+```shell
+# use the actual hash you see in your container, as well as the latest chk-XX directory
+./bin/flink run --class io.ipolyzos.compute.v4.EnrichmentStream \
+ -s checkpoints/0c83cf0320b3fc6fdcdb3d8323c27503/chk-17/ \
+ job.jar
+```
+
+With this command, we start the job while instructing Flink to use the latest checkpoint and continue from there.
+Navigate to the Flink UI and you should see something like this:
+
+
+
+You can see that while we consume new order events, the events actually get enriched with `user` and `item` information, even though our source streams haven't read any new records.
+This means the state is restored from the checkpoint and flink knows how to rebuild it without replaying all the events from the topics.
+
+## 7. Additional Resources
+- [Rock The JVM Apache Flink Course](https://rockthejvm.com/p/flink)
+- Apache Pulsar Documentation:
+ - [Pulsar Overview](https://pulsar.apache.org/docs/concepts-overview/)
+ - [Pulsar Producers](https://pulsar.apache.org/docs/concepts-messaging/)
+- StreamNative Academy:
+ - [Apache Pulsar Fundamentals](https://www.academy.streamnative.io/courses/course-v1:streamnative+AP101+UNLM/about)
+ - [Pulsar API Essentials - Java](https://www.academy.streamnative.io/courses/course-v1:streamnative+AP101-Lab+UNLM/about)
+- [Apache Pulsar Ebooks](https://streamnative.io/ebooks/)
+- [Using RocksDB State Backend in Apache Flink - When and How](https://flink.apache.org/2021/01/18/rocksdb.html)
+- [Apache Flink Restart Strategies](https://kartikiyer.com/2019/05/26/choosing-the-correct-flink-restart-strategy-avoiding-production-gotchas/)
+- [Apache Flink Checkpoints](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/state/checkpoints/)
+
+## 8. Conclusion
+
+In this long-form article, we integrated Apache Pulsar with Apache Flink:
+
+- we learned how to read data from Pulsar topics with the Pulsar connector for Flink
+- we learned how to enrich Pulsar data with `connect` and `ProcessFunction`s in Flink
+- we learned how to save missing data for investigation using _side outputs_
+- we made our jobs fault-tolerant with checkpoints
+
+Pulsar and Flink are a powerful combination of stateful data streaming, and we hope this article will help you make the best of both.
diff --git a/_posts/2022-06-21-slick.md b/_posts/2022-06-21-slick.md
new file mode 100644
index 000000000000..d9bf1c4e0429
--- /dev/null
+++ b/_posts/2022-06-21-slick.md
@@ -0,0 +1,490 @@
+---
+title: "Slick Tutorial"
+date: 2022-06-21
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, beginners]
+excerpt: "An introduction to the popular database library in Scala: Slick."
+---
+_This article is brought to you by [Yadu Krishnan](https://github.com/yadavan88), a new contributor to Rock the JVM. He's a senior developer and constantly shares his passion for new languages, libraries and technologies. He also loves writing Scala articles, especially for newcomers._
+
+This is a beginner-friendly article to get started with Slick, a popular database library in Scala. After following this post, you will be able to write Scala code to communicate with a database using SQL.
+
+This guide will explain
+- what Slick is and how to use Slick for basic CRUD operations
+- how to apply advanced concepts like join, transaction etc. using Slick
+- how to use Postgres specific data types using slick-pg
+- how to auto-generate Slick schema from database
+
+For this blog, we will build a basic database for movies and related entities. We will use Slick to save and fetch rows from multiple tables related to the movies database. For explaining different features of Slick, we will make use of tables with different types of columns.
+
+## 1. Introduction
+Slick is a functional relational library in Scala which makes working with relational databases easier. We can interact with the database almost in the same way as we do with Scala collections. Additionally, Slick uses asynchronous programming using Scala Futures. It also supports the usage of plain SQL queries which might come in handy if we want to exactly control the way the queries are built.
+Apart from that, Slick provides compile time safety by mapping the database columns to Scala data types. This ensures that it is less likely to get runtime errors for database queries.
+
+We assume the readers have basic knowledge of Scala and PostgreSQL for this post.
+
+## 2. Setup
+For this blog, we will be using Slick with _PostgreSQL_ and _Hikari_ connection pool. Also we will be using _slick-pg_ library for advanced postgres features.
+**We will be using Scala 2 version as support for Scala 3 is still in progress for Slick.**
+Let's add all the necessary dependencies together in the _build.sbt_:
+
+```scala
+libraryDependencies ++= Seq(
+ "com.typesafe.slick" %% "slick" % "3.3.3",
+ "org.postgresql" % "postgresql" % "42.3.4",
+ "com.typesafe.slick" %% "slick-hikaricp" % "3.3.3",
+ "com.github.tminglei" %% "slick-pg" % "0.20.3",
+ "com.github.tminglei" %% "slick-pg_play-json" % "0.20.3"
+)
+```
+
+We need to have a working PostgreSQL database for testing the application. Some options are:
+- Installing a PostgreSQL database locally
+- Use a dockerized PostgreSQL instance (locally). In this tutorial, you can just run `docker-compose up` to set up the database tables.
+- Use any free online services such as [ElephantSQL](https://www.elephantsql.com/)
+
+A sample `docker-compose.yaml` file looks like this:
+```
+version: '3.8'
+services:
+ db:
+ image: postgres
+ restart: always
+ environment:
+ - POSTGRES_USER=postgres
+ - POSTGRES_PASSWORD=admin
+ ports:
+ - '5432:5432'
+ volumes:
+ - db:/var/lib/postgresql11/data
+ - ./db/init-scripts.sql:/docker-entrypoint-initdb.d/scripts.sql
+
+volumes:
+ db:
+ driver: local
+```
+
+Next, we can add the database configurations to the config file such as _application.conf_.
+
+```scala
+postgres = {
+ connectionPool = "HikariCP"
+ dataSourceClass = "org.postgresql.ds.PGSimpleDataSource"
+ properties = {
+ serverName = "localhost"
+ portNumber = "5432"
+ databaseName = "postgres"
+ user = "postgres"
+ password = "admin"
+ }
+ numThreads = 10
+}
+```
+
+Within the _postgres_ node, we will be providing all the necessary configurations to connect to the database. The key _connectionPoll_ lets us use a connection pool to improve the database querying performance. If we don't provide that property, it will connect without using the connection pooling feature. Slick will be using the JDBC connection implementation from the _PGSimpleDataSource_ to connect to the database. This class is provided by the JDBC library we added, in our case _postgresql:42.3.4_. The node _properties_ are the information required to connect to the correct database.
+Alternatively, we can also use direct URL and connect to the database. For example, for connecting to the _elephantsql_ service, we can use the configuration as:
+```scala
+databaseUrl {
+ dataSourceClass = "slick.jdbc.DatabaseUrlDataSource"
+ properties = {
+ driver = "org.postgresql.Driver"
+ url = "postgres://username:password@abul.db.elephantsql.com/dbname"
+ }
+}
+```
+In this case, we are using a different dataSource as _DatabaseUrlDataSource_ which is provided by Slick itself. We then provide the connection information under the node _properties_.
+
+There are many other ways to connect to the database, which can be found [in the docs](https://scala-slick.org/doc/3.3.0/database.html).
+
+Slick uses a combination of case classes and Slick Table classes to model the database schema. For that, we need to provide a JDBC Profile. Since we are using PostgreSQL, we can use the _PostgresProfile_.
+
+As next step, we can create a connection instance using the _PostgresProfile_ and previously defined configuration.
+```scala
+import PostgresProfile.profile.api._
+object Connection {
+ val db = Database.forConfig("postgres")
+}
+```
+
+## 3. Models
+
+As mentioned in the introduction, Slick provides compile-time type safety by mapping the database fields to Scala case classes. So, we need to create a case class corresponding to the database table for us to use in the queries.
+
+```scala
+final case class Movie(
+ id: Long,
+ name: String,
+ releaseDate: LocalDate,
+ lengthInMin: Int
+)
+```
+Next, we need to create a Slick Table class. Slick Table maps the table fields to the case class. We need to import the PostgresProfile api to get the relevant methods in scope.
+
+```scala
+class SlickTablesGeneric(val profile: PostgresProfile) {
+ import profile.api._
+ class MovieTable(tag: Tag) extends Table[Movie](tag, Some("movies"), "Movie") {
+ def id = column[Long]("movie_id", O.PrimaryKey, O.AutoInc)
+ def name = column[String]("name")
+ def releaseDate = column[LocalDate]("release_date")
+ def lengthInMin = column[Int]("length_in_min")
+ override def * = (id, name, releaseDate, lengthInMin) <> (Movie.tupled, Movie.unapply)
+ }
+}
+```
+We need to first extends the class with _Table_ provided by Slick. It takes _tag_ as the first parameter. This is used by Slick to mark the class as a Table class. We can give an optional parameter for the database schema and then the table name. If schema is not provided, it will use the default schema, _public_ in the case of postgres.
+For each of the column in the table, we need to define a def with the corresponding column type of the Slick. We can use the method _column_ with the correct data type. We also need to provide the column name and any other properties that are needed. For example, let's look at the primary key **movie_id** in the Movie table.
+
+The method _column[Long]_ defines the column type. It takes the string parameter "movie_id" which is the actual column name in the database table. After that, we can provide multiple properties as vararg field.
+_O.PrimaryKey_ informs Slick that this column is the primary key.
+_O.AutoInc_ informs that this column is an auto-increment field and postgres will take care of incrementing the value.
+
+In the same way, we define the other fields.
+
+Now, we have to override a method `*` which does the mapping between the column field and the case class field. The `<>` operator maps a tuple to the case class fields.
+
+Now, we can create an instance for the _MovieTable_. We will be using this instance to write Slick queries to communicate with database.
+
+```scala
+lazy val movieTable = TableQuery[MovieTable]
+```
+We can also create a Singleton object for the _SlickTablesGeneric_ by providing the _Profile_. The _Profile_ class contains all the necessary methods for Slick to connect to the database and execute the queries.
+For now, we will be using the profile provided by Slick for the Postgres(_slick.jdbc.PostgresProfile_). Later, we can see how to write custom profile.
+
+```scala
+object SlickTables extends SlickTablesGeneric(PostgresProfile)
+```
+
+## 4. Basic CRUD Operations
+Now that we have completed the initial setup, we can look at performing basic CRUD operations using Slick. We will be using the _movieTable_ we created for all the operations. But before that, we need to manually create the database in Postgres and also create the table.
+```sql
+create extension hstore;
+create schema movies;
+create table if not exists movies."Movie" ("movie_id" BIGSERIAL NOT NULL PRIMARY KEY,"name" VARCHAR NOT NULL,"release_date" DATE NOT NULL,"length_in_min" INTEGER NOT NULL);
+create table if not exists movies."Actor" ("actor_id" BIGSERIAL NOT NULL PRIMARY KEY,"name" VARCHAR NOT NULL);
+create table if not exists movies."MovieActorMapping" ("movie_actor_id" BIGSERIAL NOT NULL PRIMARY KEY,"movie_id" BIGINT NOT NULL,"actor_id" BIGINT NOT NULL);
+create table if not exists movies."StreamingProviderMapping" ("id" BIGSERIAL NOT NULL PRIMARY KEY,"movie_id" BIGINT NOT NULL,"streaming_provider" VARCHAR NOT NULL);
+create table if not exists movies."MovieLocations" ("movie_location_id" BIGSERIAL NOT NULL PRIMARY KEY,"movie_id" BIGINT NOT NULL,"locations" text [] NOT NULL);
+create table if not exists movies."MovieProperties" ("id" bigserial NOT NULL PRIMARY KEY,"movie_id" BIGINT NOT NULL,"properties" hstore NOT NULL);
+create table if not exists movies."ActorDetails" ("id" bigserial NOT NULL PRIMARY KEY,"actor_id" BIGINT NOT NULL,"personal_info" jsonb NOT NULL);
+```
+The above scripts are for all the tables we will be using as part of this blog.
+
+### 4.1. Insert Rows
+Firstly, we can insert a record into the database.
+```scala
+val shawshank = Movie(1L, "Shawshank Redemption", LocalDate.of(1994, 4, 2), 162)
+
+def insertMovie(movie: Movie): Future[Int] = {
+ val insertQuery = SlickTables.movieTable += movie
+ Connection.db.run(insertQuery)
+}
+```
+The `db.run()` method takes an instance of `DBIOAction` and execute the action on database. DBIOAction is the most important type in Slick as all the queries in Slick are an instance of DBIOAction. DBIOAction takes 3 parameters as `DBIOAction[R,T,E]`, where `R` is the result type of the query, `T` specifies if streaming is supported or not and `E` is the effect type(Read/Write/Transaction/DDL).
+
+Now we can invoke _insertMovie(shawshank)_ and it will asynchronously insert the record into the database. Since we have used the _BIGSERIAL_ for the `movie_id`, postgres will automatically assign a numeric value.
+If we want to copy the generated id and return it along with the inserted object, we can use theee method `returning` as:
+```scala
+val insertQueryWithReturn = SlickTables.movieTable.returning(SlickTables.movieTable) += movie
+val movieFuture: Future[Movie] = db.run(insertQueryWithReturn)
+```
+
+If we want to ensure that the provided movie_id is used instead of the autogenerated one, we can use the method `forceInsert` instead of `+=`.
+
+```scala
+val forceInsertQuery = SlickTables.movieTable forceInsert movie
+```
+We can also insert multiple movies at a time.
+```scala
+val insertBatchQuery = SlickTables.movieTable ++= Seq(movie)
+val forceInsertBatchQuery = SlickTables.movieTable forceInsertAll Seq(movie)
+```
+
+### 4.2. Querying Rows from Database
+Now, let's see how to execute select queries and fetch the rows from the table. To get all the movies, we can use the `movieTable` as:
+```scala
+val movies: Future[Seq[Movie]] = db.run(SlickTables.movieTable.result)
+```
+The method `.result` on the _movieTable_ will create an executable action. We can filter the rows in the table using the _filter_ method just like on a collection. However, we need to use `===` method instead. This method will compare the value provided with that in the database column. For this method to be available, we need to import _api_ from the profile.
+```
+import profile.api._
+```
+where profile is _PostgresProfile_ we used before.
+
+```scala
+def findMovieByName(name: String): Future[Option[Movie]] = {
+ db.run(SlickTables.movieTable.filter(_.name === name).result.headOption)
+}
+```
+
+### 4.3. Update a Row
+We can use the method `update` to modify a record after applying the filter.
+```scala
+def updateMovie(movieId: Long, movie: Movie): Future[Int] = {
+ val updateQuery = SlickTables.movieTable.filter(_.id === movieId).update(movie)
+ db.run(updateQuery)
+}
+```
+In the above sample, if the filter by id is not applied, it will update all the records with the same value.
+
+If we want to update only a particular field, we can use the `map` to get the field and then update it. For example, to update the movie name, we can do as:
+```scala
+val updateMovieNameQuery = SlickTables.movieTable.filter(_.id === movieId).map(_.name).update("newName")
+```
+
+### 4.4. Delete a Row
+We can also delete a record from the database by using the method `delete`. To delete a movie we can do:
+```scala
+val deleteQuery = SlickTables.movieTable.filter(_.id === movieId).delete
+```
+
+## 5. Advanced Options
+Since we are become familiar with the basic CRUD operations, let's look at some more advanced concepts.
+
+### 5.1 Executing Plain Query
+Sometimes we might need to run plain SQL queries to get the results in a better way. Slick provides multiple ways to run the plain query. Let's look at a simple way to run the query to select rows. However, since we are using the plain query, we need to provide some additional information to Slick to make the queries typesafe. For that, we need to provide an implicit value with the relevant mappings.
+
+```scala
+def getAllMoviesByPlainQuery: Future[Seq[Movie]] = {
+ implicit val getResultMovie =
+ GetResult(r => Movie(r.<<, r.<<, LocalDate.parse(r.nextString()), r.<<))
+ val moviesQuery = sql"""SELECT * FROM public."Movie" """.as[Movie]
+ db.run(moviesQuery)
+}
+```
+
+The implicit _GetResult_ informs Slick on how to map the results of a plain query to required case class. _GetResult_ takes a lambda, which has the _ResultSet_ from the query. This implicit is used when the `as` method is applied to convert the ResultSet to case class.
+Here, we need to provide the data types of the result fields so that Slick can apply the proper type handling. We can retrieve the values from result set using _r.nextInt_, _r.nextString_ and so on. But if we need to just map the columns to case class fields without any transformation, we can simply use the method `r.<<` on the result set. The method `<<` may be considered as a placeholder for the datatype, with Slick automatically inferring the correct type. In the above example, we are explicitly parsing the date to _LocalDate_ format. We can apply any other transformations on the column result before setting the value on the case class.
+
+### 5.2. Transactional Queries
+When we have multiple queries that modifies the database table, it is always advisable to use transactions. It will ensure that the modifications happen atomically. When we use transaction, if one of the queries in the transaction fails, all the queries in the same transaction will be rolledback.
+
+```scala
+def saveWithTransaction(movie: Movie, actor: Actor): Future[Unit] = {
+ val saveMovieQuery = SlickTables.movieTable += movie
+ val saveActorQuery = SlickTables.actorTable += actor
+ val combinedQuery = DBIO.seq(saveMovieQuery, saveActorQuery)
+ db.run(combinedQuery.transactionally)
+}
+```
+The method `transactionally` on the _combinedQuery_ will make both the insert queries in a single transaction. So, if one of the fails, both the inserts will be rolled back.
+We can combine multiple queries in Slick using _DBIO.seq_. The _seq_ method takes a vararg of _DBIOAction_ which will then execute all the actions sequentially. If we don't add the method `.transactionally` at the end, it will run all the queries, but without transaction.
+
+### 5.3. Joining Tables
+
+Slick also provides methods to write join queries. Let's try to join _Actor_ table and _MovieActorMapping_ table to fetch the results.
+
+```scala
+def getActorsByMovie(movieId: Long): Future[Seq[Actor]] = {
+ val joinQuery: Query[(SlickTables.MovieActorMappingTable, SlickTables.ActorTable), (MovieActorMapping, Actor), Seq] = movieActorMappingTable
+ .filter(_.movieId === movieId)
+ .join(actorTable)
+ .on(_.actorId === _.id)
+
+ db.run(joinQuery.map(_._2).result)
+}
+```
+The above join operation returns a tuple of both the table results as a _Query_ type. We can then transform the query in the way we want before exeucting it. Her we are applying _map_ and returning only the _Actor_ table results and discarding the other one.
+
+Apart from this, Slick also provide methods for left and right outer joins as well using _joinLeft_ and _joinRight_ combinators.
+
+### 5.4. Mapping Enumeration Field to Column
+In all the above samples, we used standard data types such as _Int_, _String_, _Date_, etc for the case classes. If we want to use a custom type, we need to provide an implicit converter to convert between the Scala type and relevant column type. Let's try to use an enumeration field in case class.
+
+```scala
+object StreamingProvider extends Enumeration {
+ type StreamingProviders = Value
+ val Netflix = Value("Netflix")
+ val Hulu = Value("Hulu")
+}
+final case class StreamingProviderMapping(
+ id: Long,
+ movieId: Long,
+ streamingProvider: StreamingProvider.StreamingProviders
+)
+```
+We created enums for Streaming providers. We can then provide the enum in the case class field. We are using Scala 2 enums as there is no support yet for Scala 3.
+
+Now, let's create the mapping table for Slick. We are going to use the same format for creating the Slick table by extending with _Table_ and implementing the `*` method for mapping.
+
+```scala
+class StreamingProviderMappingTable(tag: Tag)
+ extends Table[StreamingProviderMapping](tag, Some("movies"), "StreamingProviderMapping") {
+
+ implicit val providerMapper =
+ MappedColumnType.base[StreamingProvider.StreamingProviders, String](
+ e => e.toString,
+ s => StreamingProvider.withName(s)
+ )
+
+ def id = column[Long]("id", O.PrimaryKey, O.AutoInc)
+ def movieId = column[Long]("movie_id")
+ def streamingProvider = column[StreamingProvider.StreamingProviders]("streaming_provider")
+ override def * =
+ (
+ id,
+ movieId,
+ streamingProvider
+ ) <> (StreamingProviderMapping.tupled, StreamingProviderMapping.unapply)
+}
+lazy val streamingProviderMappingTable = TableQuery[StreamingProviderMappingTable]
+```
+Here, we defined an implicit converter for _StreamingProvider_ enum. We will be saving the enum value as a string in the column. When the record is fetched, Slick will convert it to the relevant enum type using the implicit. Slick will use the _providerMapper_ to convert between case class and database column for the enum field.
+
+### 5.5. Generating DDL Scripts from Slick Tables
+Slick also provides a way to generate Data Definition Language(DDL) scripts from the Slick tables. DDL scripts explains the structure of the database using CREATE, DROP, ALTER queries and provides additional information for relationship between tables.
+This way, we can generate the table scripts and track the versions easily. This will also make sure that we can easily set up an empty database.
+
+To generate the DDL scripts, we need to first collect all the Slick tables in a collection.
+
+```scala
+val tables = Seq(movieTable, actorTable, movieActorMappingTable, streamingProviderMappingTable)
+```
+
+Then we can combine them into a Slick DDL schema using:
+```scala
+val ddl: profile.DDL = tables.map(_.schema).reduce(_ ++ _)
+```
+Now, we can invoke the method to generate the scripts:
+```scala
+SlickTables.ddl.createIfNotExistsStatements.mkString(";\n")
+```
+This will generate SQL scripts for creating all the tables we have used in our application. If we make any changes to the Slick tables, we can then again generate the DDL scripts. We can write the results to a `.sql` file and keep in the `resource` directory within the project. This will make sure that we always have the correct database structure and can create an empty database easily.
+
+## 6. Slick-Pg for Postgres
+PostgreSQL has additional powerful data types and features. But by default, Slick doesn't have support for all the advanced features of Postgres. Some of these features are not available in many of the relational databases. However, We can use a third-party library [slick-pg](https://github.com/tminglei/slick-pg) to utilise all those amazing features of Postgres in Slick with ease. We have already added the necessary dependencies in the _build.sbt_.
+
+To use it, we need to write a custom Postgres Profile and use it instead of the Slick provided _PostgresProfile_. We can mix-in the traits from slik-pg based on the required features of postgres. Let's add the support for _JSON_, _HStore_ and _Array_ data types. _HStore_ is a special datatype available in PostgreSQL database. It is used for storing key-value pair of data similar to _Map_ type in Scala.
+
+Now, let's create a new custom profile with HStore support.
+
+```scala
+trait CustomPostgresProfile
+ extends ExPostgresProfile with PgHStoreSupport {
+
+ override val api = CustomPGAPI
+ object CustomPGAPI
+ extends API
+ with HStoreImplicits
+}
+object CustomPostgresProfile extends CustomPostgresProfile
+```
+
+To create a custom profile, we need to extend with _ExPostgresProfile_ provided by Slick-PG that is a wrapper on Slick's _PostgresProfile_. To use HStore features, we need to mix-in with _PgHStoreSupport_ trait from slick-pg.
+
+For get the implicit methods, we were importing _PostgresProfile.api.__. To get the Hstore implicit methods, we need to extend the Slick's API with _HStoreImplicits_ provided by slick-pg. Then we can create a companion object for our custom profile, so that we can import it when we need to build the queries.
+
+In the same way, we can add the support for other data types such as _JSON_, _Array_ etc.
+
+```scala
+trait CustomPostgresProfile
+ extends ExPostgresProfile with PgJsonSupport with PgPlayJsonSupport
+ with PgArraySupport with PgHStoreSupport with PgDate2Support {
+ override def pgjson = "jsonb"
+ override protected def computeCapabilities: Set[slick.basic.Capability] =
+ super.computeCapabilities + slick.jdbc.JdbcCapabilities.insertOrUpdate
+
+ override val api = CustomPGAPI
+ object CustomPGAPI
+ extends API
+ with JsonImplicits
+ with HStoreImplicits
+ with ArrayImplicits
+ with DateTimeImplicits {
+ implicit val strListTypeMapper = new SimpleArrayJdbcType[String]("text").to(_.toList)
+ implicit val playJsonArrayTypeMapper =
+ new AdvancedArrayJdbcType[JsValue](
+ pgjson,
+ (s) => utils.SimpleArrayUtils.fromString[JsValue](Json.parse(_))(s).orNull,
+ (v) => utils.SimpleArrayUtils.mkString[JsValue](_.toString())(v)
+ ).to(_.toList)
+ }
+}
+object CustomPostgresProfile extends CustomPostgresProfile
+```
+In the advanced profile, we have implemented the following features:
+- Support for HStore, JSON, Array
+- Support for jsonb type which is an improved and better way to serialise JSON data in postgres
+- Implicit parameters to support Array types for conversion between Scala class and postgres json array types
+- Support for `insertOrUpdate` feature from postgres
+
+Now, we can use `CustomPostgresProfile` instead of the `PostgresProfile` to make use of these features.
+
+### 6.1. Querying from an Array Column
+
+Let's see how we can use filter query on an postgres array column. We have created a _MovieLocationsTable_ which has a _movieLocations_ array field. If we want to filter movies which was shot on any of the input locations, we can do that as:
+```scala
+val locations: List[String] = List("USA", "Germany")
+val query = SpecialTables.movieLocationsTable.filter(_.locations @& locations.bind)
+```
+The operator `@&` will return true if there is an overlap between the input list and the database column. That means, if there is atleast one common item between the column value and input list, it will return true.
+For example, assume that the database column has the countries as `[USA, Canada, Mexico]`. The above query will match and return true since _USA_ is a common value in the input `[USA, Germany]`. The method `bind` will convert the literal value into a database bind type. Bind parameters helps the database to improve the query performance by using pre-compiled prepared queries instead of re-creating separate queries each time.
+
+Some of the other popular array operators in slick-pg are:
+- `@>` will check for contains. For example, `[USA, Mexico, Canada, Germany] @> [USA, Germany]` returns true since all elements of rightside array is present in left
+- `++` will concatenate two arrays
+- `length` returns the length of the array
+
+### 6.2. Querying from an HStore Column
+
+Postgres has an extension called as `hstore`. Once it is installed, we can use the datatype _hstore_ for the columns. It is a type which stores the data as key and value pair, equivalent to an _Map_ type in Scala/Java. We can see how to filter a hstore column using slick-pg:
+
+```scala
+def getMoviesByDistributor(distributor: String): Future[Seq[MovieProperties]] = {
+ val condition = Map("distributor" -> distributor)
+ val query = SpecialTables.moviePropertiesTable.filter(_.properties @> condition.bind)
+ Connection.db.run(query.result)
+}
+```
+The operator `@>` filters the hstore column for the key _distributor_ and the input value. Similar to array, there are many other operators in hstore as well.
+
+Some of the other available operations on HStore are:
+- `@+` concatenates two hstore columns
+- `??` checks if the provided key exists in the hstore field
+- `+>` returns the value for the key provided (`'a=>x, b=>y' -> 'a'` returns `x` as the value for `a`)
+
+### 6.3. Querying from a JSON Column
+Postgres also supports JSON datatype by default. Slick-Pg also has support for querying the json columns. Let's look at it with an example:
+
+```scala
+def getActorsBornOn(year: String): Future[Seq[ActorDetails]] = {
+ Connection.db.run(
+ SpecialTables.actorDetailsTable.filter(_.personal.+>>("birthYear") === year.bind).result
+ )
+}
+```
+In the above code, we are filtering the _personal_ column which is a JSON type in _ActorDetails_ table. The method `+>>` will get the json key _birthYear_ and compares it with the value provided.
+
+Some of the other JSON Operators are:
+- `||` combines two json fields
+- `-` removes a field from json for the matching key
+
+## 7. Code Generation
+So far, we have written the Slick tables manually. If we are following a database first approach, Slick provides a way to generate the mapping tables easily. For now, we are ignoring the slick-pg types and trying to generate the Slick mappings for basic data types.
+
+We can use an sbt plugin to do that. For that, let's add the following lines to the _plugins.sbt_ file.
+```scala
+addSbtPlugin("com.github.tototoshi" % "sbt-slick-codegen" % "1.4.0")
+libraryDependencies += "org.postgresql" % "postgresql" % "42.3.4"
+```
+
+Once the sbt is refreshed, we can add the configurations in _build.sbt_:
+```scala
+slickCodegenSettings
+enablePlugins(CodegenPlugin)
+slickCodegenDatabaseUrl := "jdbc:postgresql://localhost:5432/postgres"
+slickCodegenDatabaseUser := "postgres"
+slickCodegenDatabasePassword := "admin"
+slickCodegenDriver := slick.jdbc.PostgresProfile
+slickCodegenJdbcDriver := "org.postgresql.Driver"
+slickCodegenOutputPackage := "com.rockethejvm.generated.models"
+slickCodegenCodeGenerator := { (slickModel: model.Model) => new SourceCodeGenerator(slickModel) }
+```
+
+Now, we can use the sbt command `slickCodegen`. This will generate the case classes and Slick tables. By default, the code generator will generate the file under the path `target/scala-2.13/src_managed`. It will generate both case classes and the Slick tables. It will also generate all the special relationships like primary key, foreign keys, sequences etc based on the database structure. It will also generate all the implicit _GetResult_ parameters for the plain SQL execution. This way, we can avoid manually writing most of the necessary code for mapping the database to Scala classes.
+
+We can also customise the code generator class to use advanced features like slick-pg, but we will not be looking at this as part of this blog.
+
+## 8. Conclusion
+In this article, we looked at Slick and how we can execute different types of queries using it. We also introduced some advanced features using `slick-pg`. The sample code used in this article is available [in GitHub](https://github.com/rockthejvm/slick-demo).
diff --git a/_posts/2022-08-10-zio-streams.md b/_posts/2022-08-10-zio-streams.md
new file mode 100644
index 000000000000..0fc37db8b499
--- /dev/null
+++ b/_posts/2022-08-10-zio-streams.md
@@ -0,0 +1,847 @@
+---
+title: "ZIO Streams: A Long-Form Introduction"
+date: 2022-08-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [zio, zio-streams]
+excerpt: "A comprehensive introduction to ZIO streams, a powerful abstraction in the ZIO ecosystem."
+---
+
+_This article is brought to you by
+[Mark Rudolph](https://github.com/alterationx10), a new contributor to Rock the
+JVM. He's a senior developer, who has been working with Scala for a number of
+years. He also has been diving into the ZIO ecosystem, and loves sharing his
+learnings. Mark has a lot to share, and this article is a comprehensive piece._
+
+In this post, we're going to go over an introduction to the main components of
+ZIO Streams, how to work with them when things go right, and what to do when
+things go wrong.
+
+As a toy example, we're going to take a brief foray into asynchronous
+operations, by connecting two streams to the same concurrent data structure.
+
+For a more concrete example, we are going to write a program that will parse
+markdown files, extract words identified as tags, and then regenerate those
+files with tag-related metadata injected back into them.
+
+If you're interested in learning the ZIO core library, check out the Rock the JVM [ZIO course](https://rockthejvm.com/p/zio).
+
+## Set up
+
+We're going to base this discussion off of the latest ZIO 2.0 code, which was
+officially released on June 24th, 2022. We're also using an `RC` of zio-json (at
+the time of writing, mid-June 2022), which is only used to pretty-print a `Map`
+as JSON in our of our examples. These are the dependencies to include for our
+walk through:
+
+```scala
+libraryDependencies ++= Seq(
+ "dev.zio" %% "zio" % "2.0.0",
+ "dev.zio" %% "zio-streams" % "2.0.0",
+ "dev.zio" %% "zio-json" % "0.3.0-RC8"
+)
+```
+
+## What's a Stream?
+
+_Broadly:_ A _stream_ is a series of data elements that are made available over
+time.
+
+### LazyList
+
+A specific example of a stream implementation, is Scala's `LazyList`. A
+`LazyList` is a linked list, where the `tail` is lazily evaluated. This fits our
+broad definition above, because the tail isn't evaluated until accessed -
+meaning they are made available over the time. If we investigated this in a
+_repl_, we'd see :
+
+```shell
+scala> val ll = LazyList(1,2,3,4,5)
+val ll: scala.collection.immutable.LazyList[Int] = LazyList()
+
+scala> ll.head
+val res1: Int = 1
+
+scala> ll.tail
+val res2: scala.collection.immutable.LazyList[Int] = LazyList()
+```
+
+Once the `LazyList` has been created, we can access the `head` element, but the
+`tail` is not yet computed. This is in stark comparison to a `List`, for which
+the same exercise would yield:
+
+```shell
+scala> val l = List(1,2,3,4,5)
+val l: List[Int] = List(1, 2, 3, 4, 5)
+
+scala> l.head
+val res0: Int = 1
+
+scala> l.tail
+val res1: List[Int] = List(2, 3, 4, 5)
+```
+
+Scala's `LazyList` also has the bonus of the elements being memoized - meaning
+they are only computed once. Let's go back to the _repl_, and look at our
+`LazyList` after the first two elements have been accessed.
+
+```shell
+scala> ll.tail.head
+val res3: Int = 2
+
+scala> ll
+val res4: scala.collection.immutable.LazyList[Int] = LazyList(1, 2, )
+```
+
+We can see that the first two elements now clearly show! If you haven't felt the
+excitement of generating the Fibonacci sequence with `LazyList`s, I encourage
+you to do so. However, the most important take-away from this feature, is that
+_memoization_ is not in our broad definition above - this is not a feature of
+all streams.
+
+### Logs-As-A-Stream
+
+Many messaging platforms, such as Kafka, Pulsar, and/or RabbitMQ have what they
+advertise as `Stream`s. At the heart of this, there is the concept that a
+message is _produced_, and written to disk as an _append-only_ log. _Some time
+later_ a `consumer` can read back entries from that log at their own leisure,
+and there is a guarantee that within the same offset (i.e. the first 1000
+elements), the data will be constant, even if re-processed later. This fits our
+broad definition, because the data is ordered and available over time.
+
+### FileInputStream
+
+In our example later, we are going to process blog posts to parse tag meta-data.
+These files are not processed terribly different than our append-only log above;
+any given file is an ordered collection of elements, we read them in one at a
+time, and do something with that information as we go. A notable difference from
+above, is that the elements within an offset can change _when not being
+processed_. If I go back and edit a blog post, there is no guarantee that when
+the file is re-processed that the first 1000 elements will be the same as
+before.
+
+### Stream Recap
+
+We can see that the fluid meaning of what a stream is can solidify itself around
+a context.
+
+If we're in the mindset of Kafka, a stream might mean the implication of
+consistent data within an offset _at all times_.
+
+If we're in the mindset of parsing files, a stream might imply meaningfully
+ordered elements that consistently build a larger concept, _at the time of
+processing_. This is to say, here it is more important that a stream of `Byte`
+can always be processed into a `String` by following an encoding pattern. If we
+go back and re-process it, we don't care if the contents have changed, just that
+the content can be processed.
+
+As we move on to ZIO Streams, we should keep our broad definition in mind, so we
+don't ensnare ourselves on the implementation details.
+
+## ZIO Stream Components
+
+Before we start discussing the core components of ZIO Streams, let's first
+revisit the type signature of a zio: `ZIO[R, E, A]`. This is an effect that will
+compute a single _value_ of type `A`, requires _dependencies_ of type `R` to do
+so, and could possibly fail with an _error_ of type `E`. Note, that `E` are
+errors you potentially want to recover from - an error which occurs that is not
+of type `E` is called a _defect_. If there are no dependencies required, then
+`R` is `Any`. If there are no errors you expect to recover from, then `E` is
+`Nothing`. The concept of declaring _dependencies_, _errors_, and _values_ in
+our type signature will be central to our understanding of `ZStream`s going
+forward!
+
+We should also briefly mention the `Chunk[A]`. Due to practicality, and
+performance, our streams work in batches, and the `zio.Chunk[A]` is an immutable
+array-backed _collection_, containing elements of type `A`, which will often
+present itself when working with `ZStream`s.
+
+### ZStream
+
+A ZStream has the signature: `ZStream[R, E, O]`, which means it will produce
+**some number** of `O` elements, requiring dependencies `R` to do so, and could
+possibly fail with an error of type `E`. Since this is a _stream_ we could
+possibly be producing somewhere between 0 and infinite `O` elements.
+
+A ZStream represents the _source_ of data in your work flow.
+
+### ZSink
+
+At the opposite end of our stream, we have a `ZSink`, with the signature:
+`ZSink[R, E, I, L, Z]`. `R`, and `E` are as described above. It will consume
+elements of type `I`, and produce a value of type `Z` along with any elements of
+type `L` that may be left over.
+
+A ZSink represents the terminating _endpoint_ of data in your workflow.
+
+We should mention that ZIO uses pull-based streams, meaning that elements are
+processed by being "pulled through the stream" _by_ the sink. In push-based
+systems, elements would be "pushed through the stream" _to_ the sink.
+
+Referring back to the type signature, `L` deserves a dedicated note: `L`
+describes values that have not been processed by the `ZSink`. For example, if
+our intent is to sum every element in the stream, then we would not expect any
+elements to be left over once processed:
+
+```scala
+ val sum: ZSink[Any, Nothing, Int, Nothing, Int] =
+ ZSink.sum[Int]
+```
+
+On the other hand, if our intent is to only operate on the first few elements of
+a stream, via an operation like `take`, then there exists the possibility that
+there are remaining, unprocessed values. Let's also map our output, so we can
+see the different output type.
+
+```scala
+ val take5: ZSink[Any, Nothing, Int, Int, Chunk[Int]] =
+ ZSink.take[Int](5)
+
+ val take5Map: ZSink[Any, Nothing, Int, Int, Chunk[String]] =
+ take5.map(chunk => chunk.map(_.toString))
+```
+
+If we knew our collection was finite, we could also return the leftovers that
+were not operated on - or we could outright ignore them.
+
+```scala
+ val take5Leftovers: ZSink[Any, Nothing, Int, Nothing, (Chunk[String], Chunk[Int])] =
+ take5Map.collectLeftover
+
+ val take5NoLeftovers: ZSink[Any, Nothing, Int, Nothing, Chunk[String]] =
+ take5Map.ignoreLeftover
+```
+
+### A Prelude to ZPipelines
+
+If we have some logic to _process_ a stream already, but suddenly our stream is
+a different type, we can use `contramap` to convert the input type
+appropriately. If using both `map` and `contramap`, then there is an equivalent
+`dimap` you can call as well.
+
+```scala
+ // take5Map would work on this.
+ val intStream: ZStream[Any, Nothing, Int] =
+ ZStream(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
+
+ // take5Map would not work on this.
+ val stringStream: ZStream[Any, Nothing, String] =
+ ZStream("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
+
+ // We can use contramap to use our take5 logic, and operate on stringStream with it.
+ val take5Strings: ZSink[Any, Nothing, String, Int, Chunk[String]] =
+ take5Map.contramap[String](_.toInt)
+
+// This is equivalent to take5Strings above
+ val take5Dimap: ZSink[Any, Nothing, String, Int, Chunk[String]] =
+ take5.dimap[String, Chunk[String]](_.toInt, _.map(_.toString))
+```
+
+Notice that the type of `L` is still `Int`. With `contramap`, we are operating
+on the input element to get it to the correct type (`String`), however, if
+anything is left over, then it wasn't operated on - meaning it's still the input
+type from `take5` (`Int`).
+
+It's worthwhile to pause here, and briefly discuss `contramap`. We are
+converting a value of type `A` to a value of type `B`, with a function that is
+`B => A` - this operates like `map` _in reverse_. Without diving too far into
+category theory, you can think of a Covariant Functor as something that may
+"produce" a value of type `A` (and implements a `map`), whereas a Contravariant
+Functor may "consume" a value of type `A` (and implements a `contramap`). With
+JSON as an example, a _contravariant_ `Encoder[A]` consumes a value of type `A`
+to produce JSON, whereas a _covariant_ `Decoder[A]` produces a value of type `A`
+by consuming JSON.
+
+### ZPipelines
+
+A ZPipeline converts one ZStream to another ZStream. It can conceptually be
+thought of as
+`type ZPipeline[Env, Err, In, Out] = ZStream[Env, Err, In] => ZStream[Env, Err, Out]`.
+
+The main use case of a ZPipeline is to separate out reusable transform logic,
+that can then be composed together with other ZPipelines, and then placed in
+between any appropriate ZStream and a ZSink.
+
+From our example above, let's say we wanted to sum the stream of Strings
+representing integer values. Instead of adapting our `ZSink.sum` with contramap,
+we can write:
+
+```scala
+ //stringStream.run(sum) <- This doesn't compile
+
+ val businessLogic: ZPipeline[Any, Nothing, String, Int] =
+ ZPipeline.map[String, Int](_.toInt)
+
+ val zio: ZIO[Any, Nothing, Int] =
+ stringStream.via(businessLogic).run(sum)
+```
+
+We can also _compose_ `ZPipeline`s together to form a new `ZPipeline` with
+`>>>`, for example:
+
+```scala
+ val businessLogic: ZPipeline[Any, Nothing, String, Int] =
+ ZPipeline.map[String, Int](_.toInt)
+
+ val filterLogic: ZPipeline[Any, Nothing, Int, Int] =
+ ZPipeline.filter[Int](_ > 3)
+
+ val appLogic: ZPipeline[Any, Nothing, String, Int] =
+ businessLogic >>> filterLogic
+
+ val zio: ZIO[Any, Nothing, Int] =
+ stringStream.via(appLogic).run(sum)
+```
+
+An interesting thing we're seeing here for the first time, is that connecting a
+`ZSink` to a `ZStream` results in a `ZIO`. In oder to process of our stream
+logic, we need to connect streams to sinks to produce a `ZIO` we can evaluate.
+
+Along with the typical collection-like operations you'd expect, there are a
+number of addition ones that are available directly on `ZStream`, and can be
+referenced in the
+[official docs](https://zio.dev/next/datatypes/stream/zstream/#operations) .
+
+## Handling Failures
+
+Illustrating failures can be a little lack-luster in a less-interactive medium,
+such as reading a blog post. For example, we might declare a example of stream
+with a failure as
+
+```scala
+ val failStream: ZStream[Any, String, Int] = ZStream(1, 2) ++ ZStream.fail("Abstract reason") ++ ZStream(4, 5)
+```
+
+but what does that mean in use? We're not going to intentionally instantiate a
+`ZStream` with a `.fail` in the middle of of it. Thinking about how you can get
+into a failure state is as valuable as how to recover from it. So instead of the
+example above, let's do something that feels more real, and implement an
+`InputStream` that we can control the success/failure cases.
+
+```scala
+ class RealFakeInputStream[T <: Throwable](failAt: Int, failWith: => T) extends InputStream {
+ val data: Array[Byte] = "0123456789".getBytes
+ var counter = 0
+
+ override def read(b: Array[Byte]): Int = {
+ if (counter == failAt) throw failWith
+ if (counter < data.length) {
+ b(0) = data(counter)
+ counter += 1
+ 1
+ } else {
+ -1
+ }
+ }
+
+ // Not used, but needs to be implemented
+ override def read(): Int = ???
+ }
+```
+
+Our `RealFakeInputStream` will make the elements of the string `"0123456789"`
+available. In the constructor, we can pass in a `failAt` to indicate the point
+when we should throw an exception, and `failWith` is the exception we should
+throw. By setting `failAt` higher than the length of our string, we wont fail.
+By passing in different instances of `failsWith`, we can control wether or not
+if the error is in the `E` channel of our `ZStream[R, E, O]`.
+
+With this in mind, let's set up some stream components that we can run together,
+and test different error cases. Note that we set the `chunkSize` of our
+`ZStream.fromInputStream` to 1.
+
+```scala
+ // 99 is higher than the length of our data, so we won't fail
+ val nonFailingStream: ZStream[Any, IOException, String] =
+ ZStream.fromInputStream(new RealFakeInputStream(99, new IOException("")), chunkSize = 1)
+ .map(b => new String(Array(b)))
+
+ // We will fail, and the error type matches ZStream error channel
+ val failingStream: ZStream[Any, IOException, String] =
+ ZStream.fromInputStream(new RealFakeInputStream(5, new IOException("")), chunkSize = 1)
+ .map(b => new String(Array(b)))
+
+ // We fail, but the error does not match the ZStream error channel
+ val defectStream: ZStream[Any, IOException, String] =
+ ZStream.fromInputStream(new RealFakeInputStream(5, new IndexOutOfBoundsException("")), chunkSize = 1)
+ .map(b => new String(Array(b)))
+
+ // When recovering, we will use this ZStream as the fall-back
+ val recoveryStream: ZStream[Any, Throwable, String] =
+ ZStream("a", "b", "c")
+
+ // We will pull values one at a time, and turn them into one string separated by "-"
+ val sink: ZSink[Any, Nothing, String, Nothing, String] =
+ ZSink.collectAll[String].map(_.mkString("-"))
+```
+
+Let's write a success case program to pull values through, one at a time, and
+turn them into a single String where the values are separated by "-".
+
+```scala
+object ZStreamExample extends ZIOAppDefault {
+
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ nonFailingStream
+ .run(sink)
+ .debug("sink")
+}
+```
+
+The output would look like `sink: 0-1-2-3-4-5-6-7-8-9`. As a quick aside, a new
+feature in ZIO 2 is the `.debug(prefix)` method, which will log a line
+containing the computed value of the ZIO, prefixed with `prefix: `. This is
+where the `sink: ` portion of the output comes from.
+
+Now let's turn our attention to the `failingStream`.
+
+```scala
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ failingStream
+ .run(sink)
+ .debug("sink")
+```
+
+The above code will fail less than gracefully, with something like
+
+```shell
+ sink: Fail(java.io.IOException: ,StackTrace(Runtime(6,1655962329,),Chunk(com.alterationx10.zse.ZStreamExample.run(ZStreamExample.scala:53),com.alterationx10.zse.ZStreamExample.run(ZStreamExample.scala:54))))
+timestamp=2022-06-23T05:32:09.924145Z level=ERROR thread=#zio-fiber-0 message="" cause="Exception in thread "zio-fiber-6" java.io.IOException: java.io.IOException:
+ at com.alterationx10.zse.ZStreamExample.run(ZStreamExample.scala:53)
+ at com.alterationx10.zse.ZStreamExample.run(ZStreamExample.scala:54)"
+```
+
+Let's see how we can recover with `orElse`/`orElseEither`:
+
+```scala
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ failingStream.orElse(recoveryStream)
+ .run(sink)
+ .debug("sink")
+```
+
+outputs `sink: 0-1-2-3-4-a-b-c`. From the result, we can see that the elements
+of the original stream were processed up to the point of failure, and then
+elements of the recovery stream started to be processed afterwards.
+
+```scala
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ failingStream.orElseEither(recoveryStream)
+ .run(ZSink.collectAll[Either[String, String]])
+ .debug("sink")
+```
+
+outputs
+`sink: Chunk(Left(0),Left(1),Left(2),Left(3),Left(4),Right(a),Right(b),Right(c))`.
+The interesting thing of `orElseEither` is that we can distinguish the
+transition when `failingStream` failed, and `recoveryStream` was used, as the
+former values would be a `Left` and latter values would be `Right`. Note that we
+adjusted our `ZSink` here, sone the output changed from `String` to
+`Either[String, String]`.
+
+If we want to recover from specific failures via
+`PartialFunction[E, ZStream[R1, E1, A1]]`s we can use `catchSome` ( or
+`catchSomeCause` for causes - a.k.a _defects_).
+
+```scala
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ failingStream.catchSome {
+ case _: IOException => recoveryStream
+ }
+ .run(sink)
+ .debug("sink")
+
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ defectStream.catchSomeCause {
+ case Fail(e: IOException, _) => recoveryStream
+ case Die(e: IndexOutOfBoundsException, _) => recoveryStream
+ }
+ .run(sink)
+ .debug("sink")
+```
+
+Both of the examples above result in `sink: 0-1-2-3-4-a-b-c`. We can see that
+with `catchSomeCause` we can also drill into the cause, and be able to recover
+from errors that are not included in the error channel of our `ZStream`.
+
+If we want to exhaustively recover from errors via `E => ZStream[R1, E2, A1]`)
+(and causes), we can use `catchAll` (or `catchAllCause`).
+
+```scala
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ failingStream.catchAll {
+ case _: IOException => recoveryStream
+ case _ => ZStream("x", "y", "z")
+ }
+ .run(sink)
+ .debug("sink")
+
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ defectStream.catchAllCause(_ => recoveryStream)
+ .run(sink)
+ .debug("sink")
+```
+
+Both of the examples above result in `sink: 0-1-2-3-4-a-b-c`.
+
+If we want to push our error handling downwards, we can also transform a
+`ZStream[R, E, A]` to `ZStream[R, Nothing, Either[E, A]]` via `either`.
+
+```scala
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ failingStream.either
+ .run(ZSink.collectAll[Either[IOException, String]])
+ .debug("sink")
+```
+
+will succeed, and output
+`sink: Chunk(Right(0),Right(1),Right(2),Right(3),Right(4),Left(java.io.IOException: ))`.
+We can see that we have our processed values up until the failures as `Right`s,
+ending with a `Left` of the failure, and we did not have to provide a recovery
+stream. If we felt this level of processing was good enough, we could use
+`failingStream.either.collectRight.run(sink)` to get `sink: 0-1-2-3-4`.
+
+## Async ZStreams
+
+So far, we've discussed fairly bounded streams, from defined/known collections
+of values. It won't be long before we want to "feed" data into a stream we've
+previously created. Luckily, we can create a `ZStream` from a `Queue`/`Hub`
+directly, which are asynchronous data structures in ZIO. In our example, we'll
+use a `Queue`, and with a reference to that `queue` we can offer values to it
+elsewhere. Additional, we can also create a `ZSink` from a `Queue`/`Hub`!
+
+If we can set both the _endpoint_ and the _source_ as a queue, what if we joined
+two `ZStream`s together with the _same_ queue? Let's look at a toy example,
+where we "process" one stream to sanitize values, that we then feed that into
+another stream that now doesn't have to address that concern.
+
+After our `ZStream` practice above, and a reminder that `businessLogic` converts
+`String`s to `Int`s, we might first write something like this:
+
+```scala
+ // Bad code!
+ val program: ZIO[Any, Throwable, ExitCode] = for {
+ queue <- Queue.unbounded[Int]
+ producer <- nonFailingStream.via(businessLogic)
+ .run(ZSink.fromQueue(queue))
+ result <- ZStream.fromQueue(queue)
+ .run(ZSink.sum[Int]).debug("sum")
+ } yield ExitCode.success
+```
+
+and it will _almost_ do what we think! We expected it to funnel all of the
+elements of the first `ZStream` through the second, and them sum them in the
+`ZSink.sum` to produce a resulting value. However, it will process the values,
+and feed them to the second `ZStream`, **but** our program will hang. Why?
+Because we're working with asynchronous things now. Our `result` has processed
+all the elements from our `producer` - but it continues to wait for any new
+values that could be passed into the our `queue`. It can't complete! We need to
+signal to `result` that we should finalize, by closing the `queue`. We could
+make a `Promise`, and use it to `await` before closing the `queue`, but luckily
+this functionality is already included with `ZSink.fromQueueWithShutdown`
+
+Now, we might be tempted to write:
+
+```scala
+ // Still bad code!
+ val program: ZIO[Any, Throwable, ExitCode] = for {
+ queue <- Queue.unbounded[Int]
+ producer <- nonFailingStream.via(businessLogic)
+ .run(ZSink.fromQueueWithShutdown(queue))
+ result <- ZStream.fromQueue(queue)
+ .run(ZSink.sum[Int]).debug("sum")
+ } yield ExitCode.success
+```
+
+and it will work _even less than expected_! We will be greeted by an exception,
+because our queue will be closed before we initialize the `ZStream` of our
+`result` with it! We need to think asynchronously top-to-bottom, and realize
+that we need to `fork` both our `producer` _and_ `result`, and then `join` them,
+so our `queue` is appropriately closed _after_ we've feed all of out data into
+it, and our second stream has opened from it to process the values in it.
+
+```scala
+ val program: ZIO[Any, Throwable, ExitCode] = for {
+ queue <- Queue.unbounded[Int]
+ producer <- nonFailingStream.via(businessLogic)
+ .run(ZSink.fromQueueWithShutdown(queue))
+ .fork
+ result <- ZStream.fromQueue(queue)
+ .run(ZSink.sum[Int]).debug("sum")
+ .fork
+ _ <- producer.join
+ _ <- result.join
+ } yield ExitCode.success
+```
+
+Asynchronous code can be tricky to debug, but becomes a lot easier if you
+recognize the scope of context you are working in - i.e Are we processing the
+content of a file, which is finite, or are we streaming generated events to a
+websocket, which could be infinite.
+
+## Example: Processing Files
+
+Let's walk through a simple, but working example application that parses
+markdown blog entries, automatically looks for tagged words, and re-generates a
+file with tags automatically added, and linked to their respective page. We'll
+also generate a "search" file, that is a JSON object that could be used by a
+JavaScript front end to find all pages that use a particular tag.
+
+### The Content
+
+We'd normally read these from files, but for the purpose of this post, we'll
+have the example markdown as a String containing the sample content.
+
+```scala
+ val post1: String = "hello-word.md"
+ val post1_content: Array[Byte] =
+ """---
+ |title: "Hello World"
+ |tags: []
+ |---
+ |======
+ |
+ |## Generic Heading
+ |
+ |Even pretend blog posts need a #generic intro.
+ |""".stripMargin.getBytes
+
+ val post2: String = "scala-3-extensions.md"
+ val post2_content: Array[Byte] =
+ """---
+ |title: "Scala 3 for You and Me"
+ |tags: []
+ |---
+ |======
+ |
+ |## Cool Heading
+ |
+ |This is a post about #Scala and their re-work of #implicits via thing like #extensions.
+ |""".stripMargin.getBytes
+
+ val post3: String = "zio-streams.md"
+ val post3_content: Array[Byte] =
+ """---
+ |title: "ZIO Streams: An Introduction"
+ |tags: []
+ |---
+ |======
+ |
+ |## Some Heading
+ |
+ |This is a post about #Scala and #ZIO #ZStreams!
+""".stripMargin.getBytes
+
+ val fileMap: Map[String, Array[Byte]] = Map(
+ post1 -> post1_content,
+ post2 -> post2_content,
+ post3 -> post3_content
+ )
+```
+
+We'll take note that we have `tags: []` in the post front-matter, and our
+content has words prefixed with a `#` that we want to index on. We'll also make
+a helper `Map` to represent how we'd call the file name to get the content
+later, as if we were reading from actual files.
+
+### Pipelines
+
+Let's break up our concerns, and cover a first step of collecting all the tags
+we'll need, and then use those results for the blog post regeneration. Finally,
+we'll put everything together into an application we can run start-to-end.
+
+#### Collecting tags
+
+Let's outline the process of collecting our tags. The general steps we want to
+take are:
+
+1. Filter words that match our tagging pattern
+2. Remove any punctuation, in case the last work in the sentence is tag - as
+ well as to remove the `#`.
+3. Convert our parsed tags to lowercase
+4. Put them all in a `Set[String]` to avoid duplicates.
+
+Let's also set up some helpers to looks for our `#tag`, as well as a reusable
+regex to remove punctuation when we need to.
+
+Along with our ZSink, we can make _compose_ our ZPipeline components together
+with the `>>>` operator, and bundle them together as:
+
+```scala
+
+ val hashFilter: String => Boolean =
+ str =>
+ str.startsWith("#") &&
+ str.count(_ == '#') == 1 &&
+ str.length > 2
+
+ val punctRegex: Regex = """\p{Punct}""".r
+
+ val parseHash: ZPipeline[Any, Nothing, String, String] =
+ ZPipeline.filter[String](hashFilter)
+
+ val removePunctuation: ZPipeline[Any, Nothing, String, String] =
+ ZPipeline.map[String, String](str => punctRegex.replaceAllIn(str, ""))
+
+ val lowerCase: ZPipeline[Any, Nothing, String, String] =
+ ZPipeline.map[String, String](_.toLowerCase)
+
+ val collectTagPipeline: ZPipeline[Any, CharacterCodingException, Byte, String] =
+ ZPipeline.utf8Decode >>>
+ ZPipeline.splitLines >>> // This removes return characters, in case our tag is at the end of the line
+ ZPipeline.splitOn(" ") >>> // We want to parse word-by-word
+ parseHash >>>
+ removePunctuation >>>
+ lowerCase
+
+ val collectTags: ZSink[Any, Nothing, String, Nothing, Set[String]] =
+ ZSink.collectAllToSet[String]
+
+```
+
+#### Regenerating Files
+
+Now that we have the tags for any given file, we can regenerate our blog post.
+We want to
+
+1. Automatically inject the tags into the front-matter of the content
+2. Add a link to every `#tag`, which takes us to a special page on our blog that
+ lists all posts with that tag (e.g. `[#TagWord](/tag/tagword)`).
+
+```scala
+ val addTags: Set[String] => ZPipeline[Any, Nothing, String, String] =
+ tags =>
+ ZPipeline.map[String, String](_.replace("tags: []", s"tags: [${tags.mkString(", ")}]"))
+
+
+ val addLink: ZPipeline[Any, Nothing, String, String] =
+ ZPipeline.map[String, String] { line =>
+ line.split(" ").map { word =>
+ if (hashFilter(word)) {
+ s"[$word](/tag/${punctRegex.replaceAllIn(word.toLowerCase, "")})"
+ } else {
+ word
+ }
+ }.mkString(" ")
+ }
+
+ val addNewLine: ZPipeline[Any, Nothing, String, String] =
+ ZPipeline.map[String, String](_.appended('\n'))
+
+ val regeneratePostPipeline: Set[String] => ZPipeline[Any, CharacterCodingException, Byte, Byte] =
+ ZPipeline.utf8Decode >>>
+ ZPipeline.splitLines >>> // we want to operate own whole lines this time
+ addTags(_) >>>
+ addLink >>>
+ addNewLine >>> // since we split on out new line characters, we should add one back in
+ ZPipeline.utf8Encode // Back to a format to write to a file
+
+ val writeFile: String => ZSink[Any, Throwable, Byte, Byte, Long] =
+ ZSink.fromFileName(_)
+
+```
+
+#### Building or Program
+
+With all of our processes defined, we can now build our program!
+
+```scala
+ val parseProgram: ZIO[Console, Throwable, ExitCode] = for {
+ tagMap <- ZIO.foreach(fileMap) { (k, v) =>
+ ZStream.fromIterable(v)
+ .via(collectTagPipeline)
+ .run(collectTags)
+ .map(tags => k -> tags)
+ }
+ _ <- ZIO.foreachDiscard(fileMap) { kv =>
+ Console.printLine(s"// Generating file ${kv._1}") *>
+ ZStream.fromIterable(kv._2)
+ .via(regeneratePostPipeline(tagMap(kv._1)))
+ .run(writeFile(kv._1))
+ }
+ _ <- Console.printLine("// Generating file search.json")
+ searchMap = tagMap.values.toSet.flatten.map(t => t -> tagMap.filter(_._2.contains(t)).keys.toSet).toMap
+ _ <- ZStream.fromIterable(searchMap.toJsonPretty.getBytes)
+ .run(ZSink.fromFileName("search.json"))
+ } yield ExitCode.success
+
+```
+
+We're just running these three samples serially, but if we wanted to speed up
+processing, we could do a `ZIO.foreachPar` to process files in parallel. Below
+is the content of the files we generated, and note that it's additionally
+formatted from the rules being applied to this post:
+
+> hello-world.md
+
+```md
+---
+title: "Hello World"
+tags: [generic]
+---
+
+======
+
+## Generic Heading
+
+Even pretend blog posts need a [#generic](/tag/generic) intro.
+```
+
+> zio-streams.md
+
+#####
+
+```md
+---
+title: "ZIO Streams: An Introduction"
+tags: [scala, zio, zstreams]
+---
+
+======
+
+## Some Heading
+
+This is a post about [#Scala](/tag/scala) and [#ZIO](/tag/zio)
+[#ZStreams!](/tag/zstreams)
+```
+
+> scala-3-extensions.md
+
+```md
+---
+title: "Scala 3 for You and Me"
+tags: [scala, implicits, extensions]
+---
+
+======
+
+## Cool Heading
+
+This is a post about [#Scala](/tag/scala) and their re-work of
+[#implicits](/tag/implicits) via thing like [#extensions.](/tag/extensions)
+```
+
+> search.json
+
+```json
+{
+ "zstreams": ["zio-streams.md"],
+ "implicits": ["scala-3-extensions.md"],
+ "generic": ["hello-word.md"],
+ "extensions": ["scala-3-extensions.md"],
+ "zio": ["zio-streams.md"],
+ "scala": ["scala-3-extensions.md", "zio-streams.md"]
+}
+```
+
+## Wrapping up
+
+Hopefully, after reading through this introduction, you are now more familiar
+and comfortable with `ZStream`s, how we can process data via `ZPipeline`s, and
+finalizing the processing of data to a `ZSink`s - as well as methods for
+recovering from errors, should something go wrong during processing.
+
+Additionally, I hope that the simple examples introducing the the prospect of
+working with `ZStream`s asynchronously, as well as the start-to-finish
+processing of files provides enough insight to get you started tackling your own
+real-world use cases.
diff --git a/_posts/2022-08-16-zio-http.md b/_posts/2022-08-16-zio-http.md
new file mode 100644
index 000000000000..b992b9ce8928
--- /dev/null
+++ b/_posts/2022-08-16-zio-http.md
@@ -0,0 +1,888 @@
+---
+title: "ZIO HTTP Tutorial: The REST of the Owl"
+date: 2022-09-15
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [zio, zio-http]
+excerpt: "How to set up an HTTP server with zio-http, the HTTP library in the ZIO ecosystem."
+---
+
+_This article is brought to you by
+[Mark Rudolph](https://github.com/alterationx10) - his second contribution to Rock the JVM. Mark is a senior developer, who has been working with Scala for a number of years. He also has been diving into the ZIO ecosystem, and loves sharing his learnings._
+
+>_If you want to learn more about the core ZIO library, check out the [ZIO course](https://rockthejvm.com/p/zio)._
+
+If you want the video version, check below:
+
+{% include video id="i__kKmwhUI4" provider="youtube" %}
+
+## Outline
+
+In this post, we're going to go over an introduction to the zio-http library,
+and take a look at some of the basic utilities it provides to get you up and
+running.
+
+By the end, we'll cover
+
+- basic routing
+- built-in and custom middleware
+- response streaming
+- websockets
+
+## Set Up
+
+This discussion will be based off of the latest ZIO HTTP code that supports ZIO
+2.0, which is an RC at the time of this writing (September 2022). The
+following dependencies are used:
+
+```scala
+val commonDependencies = Seq(
+ "io.d11" %% "zhttp" % "2.0.0-RC11",
+)
+```
+
+There is [code repository](https://github.com/alterationx10/the-rest-of-the-owl)
+to along with this post, if you care to take a look at the code in your IDE.
+
+Going forward, I will reference the library as `zhttp`.
+
+## Absolute Basics
+
+We're going to start off by discussing some of the basic concepts of `zhttp`,
+all based off of a fairly unassuming, self-contained program:
+
+```scala
+package com.alterationx10.troto
+
+import zhttp.http._
+import zhttp.service.Server
+import zio._
+
+object OwlServer extends ZIOAppDefault {
+
+ val port: Int = 9000
+
+ val app: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "owls" => Response.text("Hoot!")
+ }
+
+ val zApp: Http[Any, Nothing, Request, Response] =
+ Http.collectZIO[Request] { case Method.POST -> !! / "owls" =>
+ Random.nextIntBetween(3, 6).map(n => Response.text("Hoot! " * n))
+ }
+
+ val combined: Http[Any, Nothing, Request, Response] = app ++ zApp
+
+ val program: ZIO[Any, Throwable, ExitCode] = for {
+ _ <- Console.printLine(s"Starting server on http://localhost:$port")
+ _ <- Server.start(port, combined)
+ } yield ExitCode.success
+
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ program
+}
+```
+
+### Request => Response
+
+In `zhttp`, `Request`s are processed into `Response`s via implementations of a
+`sealed trait Http[-R, +E, -A, +B]`, which itself
+`extends (A => ZIO[R, Option[E], B])`. From the latter, we can quickly infer
+that `R` and `E` are the _resource_ and _Error_ channels of a `ZIO` effect, and
+we're going to be converting an `A` to a `B` effectually.
+
+There are some included type aliases to shorten this signature, however in this
+article we will try to stick to the full version. Also, note, that due to the
+first type alias, the official documentation tends to often refer to their code
+examples as an "http app" or "app" - this jargon might leak into this post as
+well.
+
+```scala
+type HttpApp[-R, +E] = Http[R, E, Request, Response]
+type UHttpApp = HttpApp[Any, Nothing]
+type RHttpApp[-R] = HttpApp[R, Throwable]
+type UHttp[-A, +B] = Http[Any, Nothing, A, B]
+```
+
+As a quick note, this section will have `R` as `Any` and `E` as `Nothing`. We
+will discuss including resources, and error handling later in the article.
+
+Let's take a moment to dig into our first endpoint:
+
+```scala
+ val app: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "owls" => Response.text("Hoot!")
+ }
+```
+
+We will use `A` and `B` here, knowing that above `A = Request` and
+`B = Response`. `Http.collect[A]` is a `PartialCollect[A]` - which behaves like
+a PartialFunction, meaning we're going to pattern match on something relating to
+`A` and return a `B`.
+
+We're matching against a `Request`, so let's look closer at the `case` statement
+above. The tricky syntax is the infixed `->` operator, so let's first look to
+the immediate _right_ of it: `!! / "owls"`. This is a `Path`, and `!!` denotes
+the root of the path (i.e. "/" - not to be confused with the `PathSyntax`
+operator `/`). On the _left_ of `->` is `Method.GET` - a `Method`. What `->`
+does, is tuple2 together the things on the left/right of it. The definition is
+
+```scala
+@inline def -> [B](y: B): (A, B) = (self, y)
+```
+
+In our case
+
+```scala
+case Method.GET -> !! / "owls" => Response.text("Hoot!")
+```
+
+and
+
+```scala
+case (Method.GET, !! / "owls") => Response.text("Hoot!")
+```
+
+should behave identically. So what's going on, is we are looking at a `Request`
+value, and matching on it's `Method` and `Path` - if they match, we will return
+our `Response`.
+
+It will be important for later, but we can reference the request `req` in our
+response, for example, via something like
+
+```scala
+case req @ Method.GET -> !! / "owls" => Response.text("Hoot!")
+```
+
+As a quick aside, `Http.collect` also internally lifts these to `Option`s, so it
+can handle the case of `None` when nothing may match.
+
+A `Method` models an HTTP request method, i.e. `GET`, `POST`, `DELETE`, etc...
+
+A `Path` models an HTTP request path. Let's take a moment to briefly outline
+`!!`, `/`, and `/:`.
+
+As mentioned above, `!!` represents the a path root, i.e. a "/".
+
+`/` is a path delimiter that starts extraction of the left-hand side (a left
+associative operator).
+
+`/:` is a path delimiter that starts extraction of the right-hand side (a right
+associative operator), and can match paths partially. For example, if we look at
+the code below:
+
+```scala
+case Method.GET -> "" /: "owls" /: name => Response.text(s"$name says: Hoot!")
+```
+
+and if we took to `curl`:
+
+```shell
+➜ the-rest-of-the-owl (main) ✗ curl http://localhost:9000/owls
+Hoot!%
+➜ the-rest-of-the-owl (main) ✗ curl http://localhost:9000/owls/Hooty
+Hooty says: Hoot!%
+➜ the-rest-of-the-owl (main) ✗ curl http://localhost:9000/owls/Hooty/The/Owl
+Hooty/The/Owl says: Hoot!%
+```
+
+We can see in the second, and third case, we're partially matching the remaining
+path, and it can capture more than just one segment representing a name!
+
+As a further note, you can't use `/` and `/:` in the same case statement, as
+left- and right-associative operators with same precedence may not be mixed.
+
+### Composing many Http[-R, +E, -A, +B]
+
+In our example, we also have:
+
+```scala
+ val zApp: Http[Any, Nothing, Request, Response] =
+ Http.collectZIO[Request] { case Method.POST -> !! / "owls" =>
+ Random.nextIntBetween(3, 6).map(n => Response.text("Hoot! " * n))
+ }
+```
+
+Note that `Http.collectZIO[Request]` behaves just like `Http.collect[Request]`,
+except here instead of returning a `Response`, we'll return a
+`ZIO[R, E, Response]`. Being ZIO users, it would make sense to see this form
+heavily in an app that relies on our _resourceful_ logic. In the example above,
+this endpoint will use the built-in `zio.Random` (which no longer needs to be
+declared in the `R` channel, as we're using ZIO 2), and `Hoot` at us 3 to 5
+times randomly, per request.
+
+We then combine `app` and `zApp` to pass to the server:
+
+```scala
+ val combined: Http[Any, Nothing, Request, Response] = app ++ zApp
+```
+
+There are four operators to compose these "HTTP applications": `++`, `<>`, `>>>`
+and `<<<`, and the behavior of each is as described from the
+[official documentation](https://zio.github.io/zio-http/docs/v1.x/dsl/http#composition-of-http-applications).
+
+> ++ is an alias for defaultWith. While using ++, if the first HTTP application
+> returns None the second HTTP application will be evaluated, ignoring the
+> result from the first. If the first HTTP application is failing with a Some[E]
+> the second HTTP application won't be evaluated.
+
+> <> is an alias for orElse. While using <>, if the first HTTP application fails
+> with Some[E], the second HTTP application will be evaluated, ignoring the
+> result from the first. If the first HTTP application returns None, the second
+> HTTP application won't be evaluated.
+
+> `>>>` is an alias for andThen. It runs the first HTTP application and pipes
+> the output into the other.
+
+> `<<<` is the alias for compose. Compose is similar to andThen. It runs the
+> second HTTP application and pipes the output to the first HTTP application.
+
+## Server
+
+At this pont, we have everything needed to start up an instance of our web
+server:
+
+```scala
+ val program: ZIO[Any, Throwable, ExitCode] = for {
+ _ <- Console.printLine(s"Starting server on http://localhost:$port")
+ _ <- Server.start(port, combined)
+ } yield ExitCode.success
+
+ override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] =
+ program
+```
+
+This is a simple entry point, and we only need to give `Server.start` a port
+(defined as `9000` above), and our _composed_ `Http[R, E, Request, Response]`.
+
+Note that `Server.start` internally calls `ZIO.never`, and will block your
+for-comprehension at that point. You should include it last, or append
+`.forkDaemon`, and provide your own logic afterwards.
+
+You can apply some configuration to the `Server` instance, however we won't
+cover this in much capacity in this article. If interested, you can see the
+official documentation
+[here](https://zio.github.io/zio-http/docs/v1.x/dsl/server/) and
+[here](https://zio.github.io/zio-http/docs/v1.x/examples/advanced-examples/advanced_server).
+
+## Next Steps
+
+At this stage, we've covered some `zhttp` basics, like pattern matching on a
+requests method and path, to run the appropriate logic. Out next steps will be
+about adding on extra functionality, like CRSF tokens and Authorization, via
+Middleware.
+
+### Middleware
+
+Broadly, the definition of _middleware_ is context dependent; in our realm of
+our discussion, if we're turning a `Request` into a `Response`, then it's
+anything we do _in the middle_ of that process. It may be something behind the
+scenes, like adding logging through the process, or more center-stage like
+modifying the request to add headers to the request. Middleware is great at
+helping de-couple/re-use business logic.
+
+Specifically, in the context of `zhttp`, a `Middleware` is a transformation
+function that converts _one_ `Http` to _another_.
+
+```scala
+type Middleware[R, E, AIn, BIn, AOut, BOut] = Http[R, E, AIn, BIn] => Http[R, E, AOut, BOut]
+```
+
+We _attach_ middleware to our `Http` via the `@@` operator. For example, we
+could update our logic to use a built-in debug `Middleware` like so:
+
+```scala
+ val wrapped: Http[Console with Clock, IOException, Request, Response] =
+ combined @@ Middleware.debug
+```
+
+and then, when running our application, we would see some debug messaging
+printed when a client interacts with our server:
+
+```shell
+[info] Starting server on http://localhost:9000
+[info] 200 GET /owls 9ms
+```
+
+In the next section, we will build our own Logging Middleware, and then look at
+a few of the other built in ones.
+
+### Logging
+
+Our custom middleware is going to log some information about the `Request`
+received, and the `Response` about to be sent back. We'll set up a new object
+`Verbose` and define a method `log` that returns a new `Middleware`, in which we
+will define the trait's `apply` method.
+
+We will be able to do this with the `.mapZIO` and `.contramapZIO` functionality
+of the `http` argument the `Middleware` takes in the `apply` method.
+
+For an http of type `Http[R, E, A, B]`, the `A` is the type of input, and `B` is
+the type of the output. In our example, `A = Request` and `B = Response`. If we
+want to do something with the _input_, like printing all of the request headers,
+we can use `.contramapZIO[R1, E1, Request]`:
+
+```scala
+http
+ .contramapZIO[R1, E1, Request] { r =>
+ for {
+ _ <- ZIO.foreach(r.headers.toList) { h =>
+ Console.printLine(s"> ${h._1}: ${h._2}")
+ }
+ } yield r
+ }
+```
+
+and if we want to do something with the _output_, like printing all of the
+response headers, we can use `.mapZIO[R1, E1, Response]`:
+
+```scala
+http
+ .mapZIO[R1, E1, Response] { r =>
+ for {
+ _ <- Console.printLine(s"< ${r.status}")
+ _ <- ZIO.foreach(r.headers.toList) { h =>
+ Console.printLine(s"< ${h._1}: ${h._2}")
+ }
+ } yield r
+ }
+```
+
+The complete example might look something like
+
+```scala
+package com.alterationx10.troto.middleware
+
+import zhttp.http._
+import zio._
+
+object Verbose {
+
+ def log[R, E >: Throwable]
+ : Middleware[R, E, Request, Response, Request, Response] =
+ new Middleware[R, E, Request, Response, Request, Response] {
+
+ override def apply[R1 <: R, E1 >: E](
+ http: Http[R1, E1, Request, Response]
+ ): Http[R1, E1, Request, Response] =
+ http
+ .contramapZIO[R1, E1, Request] { r =>
+ for {
+ _ <- Console.printLine(s"> ${r.method} ${r.path} ${r.version}")
+ _ <- ZIO.foreach(r.headers.toList) { h =>
+ Console.printLine(s"> ${h._1}: ${h._2}")
+ }
+ } yield r
+ }
+ .mapZIO[R1, E1, Response] { r =>
+ for {
+ _ <- Console.printLine(s"< ${r.status}")
+ _ <- ZIO.foreach(r.headers.toList) { h =>
+ Console.printLine(s"< ${h._1}: ${h._2}")
+ }
+ } yield r
+ }
+
+ }
+
+}
+```
+
+We're not modifying the input/output here, so the types remain the same and
+their values un-altered. We used `.contramapZIO` to accesses the `Request`,
+print some information about it, and then return the un-altered value. We then
+did the same thing with `mapZIO` for the `Response`.
+
+This is a very simple example, but is very illustrative of how you can easily
+update the `Request`/`Response` values if desired, or even fail-fast if a
+particular header is missing, or unverified.
+
+We attach our custom `Middleware` just as before:
+
+```scala
+val wrapped: Http[Any,Throwable,Request,Response] =
+ combined @@ Verbose.log
+```
+
+and if we run our server, and make a request via curl, in our console we should
+see something like:
+
+```shell
+[info] Starting server on http://localhost:9001
+[info] > POST /owls Http_1_1
+[info] > Host: localhost:9001
+[info] > User-Agent: curl/7.79.1
+[info] > Accept: */*
+[info] < Ok
+[info] < content-type: text/plain
+```
+
+### CORS
+
+CORS stands for _Cross-Origin Resource Sharing_. It is a mechanism to allow a
+website to allow traffic from only certain origins. For example, if we had
+`https:///my-site.com` without any CORS config, then someone at
+`https://their-site` could load our images, content, etc (and we'd get billed
+for the data usage). By applying a CORS configuration, we can make sure our
+resources are loaded when the `Origin` header is set to `my-site.com`. Requests
+trying to render from the other page would then get rejected!
+
+In addition to this, CORS can help with verifying you can interact with the
+server at all! Wouldn't it be nice to know you can't upload a 50mb png to a site
+_before_ you sent the request? You can send a CORS (`OPTIONS`) _preflight
+request_ that says "Hey, I'm from _this_ origin, and I would like to _POST_ you
+a file with some _MIME type_ that's _this big_ at _this_ endpoint. We good?" And
+if your pre-flight request succeeds, you know you can make the _actual_
+request - but if it fails, you don't have to waste the time/traffic finding it
+out.
+
+Many browsers today will automatically try to make pre-flight queries when
+content is being loaded from a domain different that the host being accessed,
+and if there is no CORS policy returned from the server, then the resources
+won't be loaded. This even means between `sub.my-domain.com` and
+`sub2.my-domain.com`.
+
+To use the built-in CORS, you need to instantiate a `CorsConfig` such as:
+
+```scala
+ val config: CorsConfig =
+ CorsConfig(
+ anyOrigin = false,
+ anyMethod = false,
+ allowedOrigins = s => s.equals("localhost"),
+ allowedMethods = Some(Set(Method.GET, Method.POST))
+ )
+```
+
+and provide it as an argument via `@@ Middleware.cors(config)`.
+
+If we look at the output of an example using the above config, we can see that
+if we send an `Origin` header that doesn't match our config's `allowedOrigins`,
+no CORS headers are returned - thus a browser would block the request. This
+would be the same as if we sent no `Origin` header.
+
+```shell
+➜ the-rest-of-the-owl (main) ✗ curl -v --header "Origin: somewhere" http://localhost:9001/owls
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+> GET /owls HTTP/1.1
+> Host: localhost:9001
+> User-Agent: curl/7.79.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< content-type: text/plain
+< set-cookie: x-csrf-token=e9250e77-49a8-4b75-bac0-27307980afba
+< content-length: 5
+<
+* Connection #0 to host localhost left intact
+Hoot!%
+```
+
+If we make the request again, with a valid `Origin` header, we can see the CORS
+`access-control-` headers are returned.
+
+```shell
+➜ the-rest-of-the-owl (main) ✗ curl -v --header "Origin: localhost" http://localhost:9001/owls
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+> GET /owls HTTP/1.1
+> Host: localhost:9001
+> User-Agent: curl/7.79.1
+> Accept: */*
+> Origin: localhost
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< content-type: text/plain
+< set-cookie: x-csrf-token=d4f7ddc2-63a7-4b78-b99f-1f9b6d6a2d0e
+< access-control-expose-headers: *
+< access-control-allow-origin: localhost
+< access-control-allow-methods: GET,POST
+< access-control-allow-credentials: true
+< content-length: 5
+<
+* Connection #0 to host localhost left intact
+Hoot!%
+```
+
+Note that the `CorsConfig` case class has a lot of default values provided, and
+it may be unintuitive at first:
+
+```scala
+object Cors {
+ final case class CorsConfig(
+ anyOrigin: Boolean = true,
+ anyMethod: Boolean = true,
+ allowCredentials: Boolean = true,
+ allowedOrigins: String => Boolean = _ => false,
+ allowedMethods: Option[Set[Method]] = None,
+ allowedHeaders: Option[Set[String]] = Some(
+ Set(HttpHeaderNames.CONTENT_TYPE.toString, HttpHeaderNames.AUTHORIZATION.toString, "*"),
+ ),
+ exposedHeaders: Option[Set[String]] = Some(Set("*")),
+ )
+}
+```
+
+For example, you might start with
+`CorsConfig(allowedOrigins = _ == "myhost.com")`, but the `anyOrigin` value is
+defaulted to `true`.
+
+### CSRF
+
+CSRF stands for _Cross-Site Request Forgery_. At a broad level, this is when
+some _nefarious code_ tries to trick you into performing an action with your
+_already logged in_ credentials. For example, let's say you were logged into a
+popular online store, and a browser plugin went rouge. Opening the plugin takes
+you to a link that _actually_ triggers an email change for your account with the
+store via a hidden form - which your browser happily send along your sessions
+information, giving a chance for an attacker to take over your account. CSRF
+tries to help stop/lessen that attack vector.
+
+CSRF involves the generation of a fairly secure/unique string (we'll call it a
+`token`), and submitting it back to our trusted site when we send information,
+as an extra level of verification that it was intentionally sent by the user.
+
+A popular way to do this is called _Double Submit Cookie_. This means that a
+secure http-only (i.e. browser javascript cannot access this!) cookie is set
+with the value of the `token`, and any routes you want protected will need to
+_both_ send this cookie, _as well as_ the `token` as a parameter. You could
+imagine that a trusted web page that is rendered with form submissions already
+have this value on a hidden input pre-filled and sets a cookie. When the form is
+submitted, the cookie will go along with the values, and the back-end server can
+verify that they are preset _and_ match! From here, you can also take it a step
+further and encrypt the cookie, so the back-end can verify that it can decrypt
+the `token` as well to ensure authenticity.
+
+`zhttp` includes `Middleware.csrfGenerate()` and `Middleware.csrfValidate()` as
+built-in options. For our example, we'll split these and add the `csrfGenerate`
+to out `Http` that has the `GET` routes, and the `csrfValidate` to our `POST`:
+
+```scala
+ val app: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "owls" => Response.text("Hoot!")
+ case Method.GET -> "owls" /: name /: !! =>
+ Response.text(s"$name says: Hoot!")
+ } @@ Middleware.csrfGenerate()
+
+ val zApp: Http[Any, Nothing, Request, Response] =
+ Http.collectZIO[Request] { case Method.POST -> !! / "owls" =>
+ Random.nextIntBetween(3, 6).map(n => Response.text("Hoot! " * n))
+ } @@ Middleware.csrfValidate()
+```
+
+Let's interact and inspect with these endpoint via curl:
+
+```shell
+➜ ~ curl -X GET -v http://localhost:9001/owls
+Note: Unnecessary use of -X or --request, GET is already inferred.
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+> GET /owls HTTP/1.1
+> Host: localhost:9001
+> User-Agent: curl/7.79.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< content-type: text/plain
+< set-cookie: x-csrf-token=2075bc8b-c64b-494c-8249-c3a87ca72fcd
+< content-length: 5
+<
+* Connection #0 to host localhost left intact
+Hoot!%
+```
+
+We made a `GET` request, and we can see that the server told us to set a cookie
+with our `x-csrf-token`.
+
+If we try to access our `POST` without the `token`, we will get a
+`403 Forbidden`!
+
+```shell
+➜ ~ curl -X POST -v http://localhost:9001/owls
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+> POST /owls HTTP/1.1
+> Host: localhost:9001
+> User-Agent: curl/7.79.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 403 Forbidden
+< content-length: 0
+<
+* Connection #0 to host localhost left intact
+```
+
+This middleware does use the _Double Submit Cookie_ method, so if we make a
+`POST` including our `token` as a cookie, _and_ a corresponding header, then we
+can obtain access to our endpoint.
+
+```shell
+➜ ~ curl -X POST -v --cookie "x-csrf-token=2075bc8b-c64b-494c-8249-c3a87ca72fcd" -H "x-csrf-token: 2075bc8b-c64b-494c-8249-c3a87ca72fcd" http://localhost:9001/owls
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+> POST /owls HTTP/1.1
+> Host: localhost:9001
+> User-Agent: curl/7.79.1
+> Accept: */*
+> Cookie: x-csrf-token=2075bc8b-c64b-494c-8249-c3a87ca72fcd
+> x-csrf-token: 2075bc8b-c64b-494c-8249-c3a87ca72fcd
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< content-type: text/plain
+< content-length: 18
+<
+* Connection #0 to host localhost left intact
+Hoot! Hoot! Hoot! %
+```
+
+### Basic Auth
+
+Basic auth is used for hiding a site behind a simple user/password check. The
+credentials are _base64 encoded_, but not encrypted, so it shouldn't be used
+except over https.
+
+We'll add a _super secret_ route to our app using the built-in
+` Middleware.basicAuth`.
+
+```scala
+ val authApp: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "secret" / "owls" =>
+ Response.text("The password is 'Hoot!'")
+ } @@ Middleware.basicAuth("hooty", "tootie")
+```
+
+In our example, we use
+
+```scala
+final def basicAuth(u: String, p: String): HttpMiddleware[Any, Nothing]
+```
+
+which takes two parameters, a username and a password. When a request comes in,
+and the credentials are present on the HTTP header, it will compare those values
+to the username and password for validation. If you wanted to pass in your own
+logic for this, you could use
+
+```scala
+case class Credentials(uname: String, upassword: String)
+final def basicAuth(f: Credentials => Boolean): HttpMiddleware[Any, Nothing]
+final def basicAuthZIO[R, E](f: Credentials => ZIO[R, E, Boolean]): HttpMiddleware[R, E]
+```
+
+We will not cover it in this article, however there are other helpful pre-made
+`MiddleWare`, for bearer tokens and custom authorization:
+
+```scala
+final def bearerAuth(f: String => Boolean): HttpMiddleware[Any, Nothing]
+final def bearerAuthZIO[R, E](f: String => ZIO[R, E, Boolean]): HttpMiddleware[R, E]
+ final def customAuth(
+ verify: Headers => Boolean,
+ responseHeaders: Headers = Headers.empty,
+ responseStatus: Status = Status.Unauthorized,
+ ): HttpMiddleware[Any, Nothing]
+ final def customAuthZIO[R, E](
+ verify: Headers => ZIO[R, E, Boolean],
+ responseHeaders: Headers = Headers.empty,
+ responseStatus: Status = Status.Unauthorized,
+ ): HttpMiddleware[R, E]
+```
+
+Looking back at out basic auth example, if we try to access our endpoint without
+credentials, we'll get a 401 Unauthorized, and a polite `www-authenticate`
+header indicating that we may be able to access it via `Basic` auth.
+
+```shell
+➜ ~ curl -v http://localhost:9001/secret/owls
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+> GET /secret/owls HTTP/1.1
+> Host: localhost:9001
+> User-Agent: curl/7.79.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 401 Unauthorized
+< www-authenticate: Basic
+< content-length: 0
+<
+* Connection #0 to host localhost left intact
+```
+
+We'll use the power of curls `--user` flag to do the encoding for us this time:
+
+```shell
+➜ ~ curl -v --user hooty:tootie http://localhost:9001/secret/owls
+* Trying 127.0.0.1:9001...
+* Connected to localhost (127.0.0.1) port 9001 (#0)
+* Server auth using Basic with user 'hooty'
+> GET /secret/owls HTTP/1.1
+> Host: localhost:9001
+> Authorization: Basic aG9vdHk6dG9vdGll
+> User-Agent: curl/7.79.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< content-type: text/plain
+< content-length: 23
+<
+* Connection #0 to host localhost left intact
+The password is 'Hoot!'%
+```
+
+We can see that our `Authorization: Basic aG9vdHk6dG9vdGll` was sent in plain
+text (base64 encoded), as well as that our request was successful.
+
+### Juggling Middleware Priority
+
+At this point, we've tacked on a few pieces of `Middleware`. Combining your
+routes can be very tricky, so let's address some issues. Let's looks at what we
+have so far:
+
+```scala
+ val app: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "owls" => Response.text("Hoot!")
+ case Method.GET -> "owls" /: name /: !! =>
+ Response.text(s"$name says: Hoot!")
+ } @@ Middleware.csrfGenerate()
+
+ val zApp: Http[Any, Nothing, Request, Response] =
+ Http.collectZIO[Request] { case Method.POST -> !! / "owls" =>
+ Random.nextIntBetween(3, 6).map(n => Response.text("Hoot! " * n))
+ } @@ Middleware.csrfValidate()
+
+ val authApp: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "secret" / "owls" =>
+ Response.text("The password is 'Hoot!'")
+ } @@ Middleware.basicAuth("hooty", "tootie")
+
+ val combined: Http[Any, Nothing, Request, Response] = app ++ authApp ++ zApp
+```
+
+We have to add `authApp` _before_ `zApp`, or our basic auth route won't be
+available! _Why is that?_ It's because we aren't sending in any CSRF validation
+tokens! Because we've added `Middleware.csrfValidate()` to `zApp`, that portion
+_happily succeeds_ in handing us back a 403 Forbidden when we don't send the
+CSRF data - thus if our `authApp` were _after_ it, we'd never reach it.
+
+The same situation would occur if we added `authApp` to the front - everything
+afterwards would also require basic auth. This also makes it not possible to
+apply a _second_ `Middleware.basicAuth("hooty2", "tootie2")` at an `Http` passed
+the first, because we'd always fail the credential validation at the first
+middleware evaluation of the credentials (it would check for `user == hooty` and
+`password == tootie`). The best we could do with basic auth is allow a set of
+users/passwords that all have the same level of access to various protected
+routes, _but not fine-grained individual access per route_.
+
+## Extra Credit
+
+Congratulations on making it this far! By now, we've set up a web server that's
+returning content, and added basic levels of security like authorization, CSRF
+tokens, and CORS policies 🎉 In the next, and final section, we will quickly
+visit websocket support and response streaming.
+
+### Websockets
+
+Websockets are also created using `Http`, but instead of collecting `Request`s,
+we typically collect `WebSocketChannelEvent`. Communication happens over the
+channel, so instead of returning `Response` we will return `Unit`. At a lower
+level, there is `Channel[A]`, which allows sending arbitrary messages of type
+`A`.There is also`ChannelEvent`, which encapsulates the types of messages that
+can be sent/received.
+
+```scala
+final case class ChannelEvent[-A, +B](channel: Channel[A], event: ChannelEvent.Event[B])
+```
+
+`WebSocketChannelEvent` is actually a type alias for
+`ChannelEvent[WebsocketFrame, WebSocketFrame]`.
+
+In our code example below, in addition to logging on some connections hooks, we
+mainly return back the data sent from the user, but formatted differently. You
+could further pattern match on the incoming message to perform different actions
+based on the incoming payload - our bundled multiple `Http`s with different
+endpoints to handle different functionality.
+
+```scala
+ val sarcastically: String => String =
+ txt =>
+ txt.toList.zipWithIndex.map { case (c, i) =>
+ if (i % 2 == 0) c.toUpper else c.toLower
+ }.mkString
+
+ val wsLogic: Http[Any, Throwable, WebSocketChannelEvent, Unit] =
+ Http.collectZIO[WebSocketChannelEvent] {
+
+ case ChannelEvent(ch, ChannelRead(WebSocketFrame.Text(msg))) =>
+ ch.writeAndFlush(WebSocketFrame.text(sarcastically(msg)))
+
+ case ChannelEvent(ch, UserEventTriggered(event)) =>
+ event match {
+ case HandshakeComplete => ZIO.logInfo("Connection started!")
+ case HandshakeTimeout => ZIO.logInfo("Connection failed!")
+ }
+
+ case ChannelEvent(ch, ChannelUnregistered) =>
+ ZIO.logInfo("Connection closed!")
+
+ }
+```
+
+```scala
+ val wsApp: Http[Any, Nothing, Request, Response] = Http.collectZIO[Request] {
+ case Method.GET -> !! / "ws" => wsLogic.toSocketApp.toResponse
+ }
+```
+
+We will use a handy cli tool called `websocat` to test our websockets. It will
+allow us to connect to our server, send messages, and see the responses that
+come back.
+
+```shell
+➜ ~ websocat ws://localhost:9002/ws
+Hello
+HeLlO
+Sarcasm is hard to convey on the internet
+SaRcAsM Is hArD To cOnVeY On tHe iNtErNeT
+```
+
+### Streaming
+
+Streaming responses is handled via `ZStream`, and works straight forwardly, if
+you are comfortable with that topic. If you would like to dig into `ZStream`s a
+bit further, I suggest you check out this
+[article](https://blog.rockthejvm.com/zio-streams/).
+
+In our example below, we take a `String` sentence, and repeat it 1000 times to
+bulk it up a bit, being sure to use the `HTTP_CHARSET` encoding when we create a
+`Chunk` for it. At this point, it's as easy as
+`HttpData.fromStream(ZStream.fromChunk(data))`! Anything you can
+`ZStream.from...` is fair game to stream, which is fairly powerful.
+
+```scala
+ val content: String =
+ "All work and no Play Framework makes Jack a dull boy\n" * 1000
+
+ val data: Chunk[Byte] = Chunk.fromArray(content.getBytes(HTTP_CHARSET))
+
+ val stream: Http[Any, Nothing, Request, Response] = Http.collect[Request] {
+ case Method.GET -> !! / "stream" =>
+ Response(
+ status = Status.Ok,
+ headers = Headers.contentLength(data.length.toLong),
+ data = HttpData.fromStream(ZStream.fromChunk(data))
+ )
+ }
+```
+
+## Wrapping Up
+
+Hopefully, after reading this introduction, you feel comfortable enough to spin
+up your own web server with zio-http, start adding on built in security
+features - as well as custom middleware logic, and delve into high performance
+via streaming responses and real-time communication via websockets.
diff --git a/_posts/2022-08-29-sbt-tutorial.md b/_posts/2022-08-29-sbt-tutorial.md
new file mode 100644
index 000000000000..4d2ce2b95ab7
--- /dev/null
+++ b/_posts/2022-08-29-sbt-tutorial.md
@@ -0,0 +1,575 @@
+---
+title: "An introduction to SBT"
+date: 2022-09-12
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, beginners, sbt]
+excerpt: "Learn how to set up and configure your Scala projects with SBT in this long-form tutorial."
+---
+
+_This article is brought to you by [Yadu Krishnan](https://github.com/yadavan88). He's a senior developer and constantly shares his passion for new languages, libraries and technologies. After his [long-form Slick tutorial](/slick), he's coming back with a new comprehensive introduction to SBT. Please enjoy!_
+
+> _This tutorial complements Rock the JVM's premium [Scala masterclass](https://rockthejvm.com/p/the-scala-bundle), as you learn to set up and configure your Scala projects._
+
+## 1. Introduction
+
+SBT is the most popular build tool in the Scala ecosystem. SBT provides a very rich DSL to configure a Scala project. It is also extensible and supports customised plugins to enhance the project.
+
+_(Daniel's note: SBT was originally an acronym for "simple build tool", but in time "simple" was replaced by "Scala". Should've been the other way around.)_
+
+In this article we'll start an SBT project from scratch, and walk through essential SBT features you need.
+
+## 2. Installation
+
+There are multiple ways to install SBT:
+- Manual Installation using the executable depending on the OS
+- Using Coursier (https://www.scala-lang.org/download/)
+- Using IDEs like IntelliJ IDEA with SBT plugin
+
+## 3. Basics of SBT
+The core of SBT is a file named _build.sbt_. We can provide the needed configuration options within _build.sbt_.
+
+### 3.1. Simplest SBT project
+
+Let's look at a simple SBT project.
+First, let's create an empty directory. We can then create an empty file within the directory as _build.sbt_.
+In _build.sbt_, let's configure the Scala version of the project we are going to use:
+
+```
+scalaVersion := "2.13.8"
+```
+Note that SBT uses a special operator `:=` to assign the value.
+
+Now, within the same directory, we can just start the SBT session by using the command `sbt`.
+If everything is fine, this will load an SBT console successfully. We can execute the command `project` within the SBT console to verify and view the SBT project name.
+
+Now, let's look at the other configuration options in the _build.sbt_ file.
+We can provide a version for the project using the option `version` :
+```
+version := "1.0"
+```
+> Note that if any change is made to the _build.sbt_, we need to exit the existing SBT console and restart it for the changes to take effect. Alternatively, we use the SBT command `reload` to forcefully apply the changes.
+
+We can set the name of the project using `name` and set an organisation for the project:
+```
+name := "rockthejvm"
+organization := "com.rockthejvm"
+```
+Once we add a name to the project, SBT console will use the project name when we start the sbt. If it is not provided, the directory name is used instead.
+
+### 3.2. Project Structure
+A simple standard SBT project follows this structure (assume that the directory is _simpleProject_):
+
+```
+simpleProject
+ |-- src
+ |--- main/scala
+ |--- test/scala
+ |-- project
+ |-- build.sbt
+ ```
+
+The Scala code and configurations are placed under _main_ or _test_ subdirectories. The _project_ is an optional directory that contains additional setting files needed for configuring the SBT project.
+
+### 3.3. SBT versions
+By default, SBT uses the installed SBT version for the projects. We can explicitly provide a specific version of the SBT. This is generally followed since there might be some breaking changes in the SBT releases and that might unknowingly affect the entire project.
+We can specify the SBT version for the project by using a file called as _build.properties_ under _project_ directory. We can mention the SBT version in the file as:
+```
+sbt.version=1.6.1
+```
+This will ensure that the project will use the SBT version as _1.6.1_ even though you might have installed another version of SBT in your machine.
+
+### 3.4. Small Scala File and SBT Commands
+Now, let's try to add a simple Scala file a main method. We will keep this file under the directory _src/main/scala_ with package as _com.rockthejvm_:
+
+```
+package com.rockthejvm
+object Main {
+ def main(args: Array[String]): Unit = {
+ println("Hello, World!")
+ }
+}
+```
+Let's start the SBT console by executing the command `sbt` at the root of the project.
+To compile the project, we can use _compile_ in the command. This will compile all the Scala files in the project and creates .class files. These files are kept under the directory _target_.
+**SBT uses _incremental compilation_ to speed up the compilation. In this way, SBT will compile only the modified files, and re-use the previously compiled class files for the rest of the files.**
+We can use the _clean_ command to remove all the generated class files.
+
+We can run the application by using the command _run_. If we have multiple main classes in the project, SBT will show a prompt to select the main class to run.
+In such case, we can use _runMain_ command by passing the class to run as:
+```
+runMain com.rockthejvm.Main
+```
+
+SBT also allows automatic compilation on any source file change. For that, we can start the compile command prefixed by `~` symbol within the SBT shell:
+```
+~compile
+```
+Now, SBT will track the source files and if any change is detected in the files, it will automatically recompile them.
+
+## 4. Adding external Dependencies
+So far, we have created Scala files without using any external dependencies. Now, let's look at how we can add additional libraries to our project. We can add the dependencies to our project by adding it to the settings _libraryDependencies_. SBT follows Ivy style format to define the dependencies as :
+```
+libraryDependencies += "com.lihaoyi" %% "fansi" % "0.4.0"
+```
+
+The method `+=` appends the provided library to the project dependencies.
+An SBT dependency contains mainly 3 parts separated by % symbol.
+The first part is the groupId of the library. The second part is the library name(artifactId) and the third part is the version of the library to be used. If you notice, you can see that a double percentage symbol (`%%`) is used between groupId and artifactId.
+Scala is not binary compatible with different versions (such as 2.11, 2.12, 2.13 etc) except for Scala 3 series. Hence there are separate releases for each Scala libraries for each required versions. %% symbol ensures that SBT uses the same Scala version of library as the project.
+That means SBT will automatically append the Scala version of the project before trying to find the library. The above dependency code is equivalent to the following format(note that single % is used, but artifact id contains the Scala major version):
+```
+libraryDependencies += "com.lihaoyi" % "fansi_2.13" % "0.4.0"
+```
+
+SBT also has triple % symbol (%%%), which is used to resolve ScalaJS libraries.
+
+We can add multiple dependencies together by using ++= and Seq :
+```
+libraryDependencies ++= Seq(
+ "com.lihaoyi" % "fansi_2.13" % "0.4.0"
+ //Can add more dependencies here.
+)
+```
+
+## 5. Scala files under project directory
+SBT allows us to keep supporting files(i.e. non-source files, not part of the project "code" itself) for the build as Scala files. This helps to configure most of the required configurations in the known format of Scala classes. We can place the Scala files under the directory _project_.
+Generally, depending on the project complexity, we keep various info (e.g. library dependencies) as object in these Scala files; they are accessible directly inside _build.sbt_. This is done so that we can keep and track all the dependencies in a single place, and access them in multiple parts (modules) of the project (more on modules shortly).
+
+## 6. SBT Console
+SBT also can start a REPL session within the project using the command `sbt console` or executing the command `console` within an SBT session. This will start up a REPL session just like the Scala REPL. However, the SBT console REPL will also load all the dependencies jars to the classpath so that we can directly import the classes and use them.
+For example, let's try to use the fansi library in the SBT console.
+We can then start up the SBT session by using `sbt` command. Then we can use the command _console_ within the SBT session to start a REPL.
+Now, within this REPL, we have access to the methods from fansi. We can execute this code:
+```
+val redString: fansi.Str = fansi.Color.Red("Hello FAnsi String!")
+```
+_Console_ is a class from the fansi library to give color to the string.
+
+In the same way, we can access any of the libraries within the project inside the REPL. This is really helpful in trying out small pieces of code within a project easily.
+
+## 7. Running Tests
+We normally write unit test cases and places them under the relevant package in the directory _src/test/scala_. Let's add _scalatest_ dependency and write a very simple test
+```
+libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.13" % Test
+```
+
+Note the special identifier _Test_ after the last % symbol. Adding `Test` informs SBT that this library is needed only for the test cases and accessible under _src/test/scala_ directory. Whenever we make the packaged application from our project, these libraries are not added to the package as they are not needed at the application run time.
+Instead of the identifier _Test_, we can also use the string `"test"` (wrapped in double quotes). However, this practice is discouraged as it doesn't provide type safety.
+
+Now that we added the dependency, we can add a test file as:
+```
+package com.rockthejvm
+import org.scalatest.funsuite.AnyFunSuite
+class SimpleTest extends AnyFunSuite {
+ test("comapare 2 strings ignoring case") {
+ val calculatedString = "ROCKtheJVM"
+ val expectedString = "rockthejvm"
+ assert(calculatedString.toLowerCase == expectedString)
+ }
+}
+```
+
+Now, we can run the tests in the project using the command `test`. This will run all the tests that are available in the project under the _src/test/scala_ path.
+
+It is also possible to run a subset of tests instead of all. For example, let's try to run only one test file. For this we will first start the SBT session. Then within the session we can run:
+```
+test:testOnly com.rockthejvm.SimpleTest
+```
+This will test only the _SimpleTest_ file. Note that we need to provide the full package path to resolve the class correctly.
+We can also use a wildcard to make it easy to run the test:
+```
+test:testOnly *SimpleTest
+```
+This will run all the classes that ends with _SimpleTest_ within the entire project irrespective of the package.
+
+So far, we started the session and then ran the test. We can do that directly from the terminal by using the command:
+```
+sbt test
+```
+This will start the SBT session and then run the test. However, we need to wrap the arguments after _sbt_ if there are some special characters are involved. For example, to run the test using wildcard, we need to use the command:
+```
+sbt 'test:testOnly *SimpleTest'
+```
+We can also use double quotes(") instead of single quotes('). This will ensure that all the arguments are passed to the SBT session at once.
+
+We can compile only the test classes by using the command `test:compile` within sbt session.
+
+## 8. Advanced configuration in _build.sbt_
+In the previous section, we saw configurations like _name_, _organization_, etc. The values we set there are applicable for the entire project. SBT also lets us do some configurations per scope.
+For example, by default SBT runs all the tests in parallel. If we somehow want to avoid that and run the tests in sequential way, we can set the config option for that. SBT uses the configuration key _parallelExecution_ for that and it is under the scope of Test.
+
+So, in _build.sbt_, we can use the setting as:
+```
+Test / parallelExecution := false
+```
+This will set the value to false within scopes of test.
+
+There are many such configurations which are configured using the scope. You can think of this something in the lines of _Test.parallelExecution_.
+
+The list of different SBT scopes are `Test`, `IntegrationTest`, `RunTime`, `Compile`, `Provided`, `Optional`, `CompileInternal` and `ScalaTool` . So, we can set the configuration only for a particular scope. For example, we can set some configurations only for the scope `IntegrationTest` as:
+```
+IntegrationTest / testOptions += Tests.Argument("-verbosity", "1")
+```
+
+## 9. Multi Module Project
+So far we have used a single module project. SBT allows to create multi-module project. This way, we can clearly separate different modules, but we can aggregate and combine different modules together to make a big project. Let's create a simple multi-module project first. For that, we can create a _build.sbt_ file as below within a new empty directory.
+
+```
+ThisBuild / scalaVersion := "2.13.8"
+ThisBuild / version := "1.0"
+ThisBuild / name := "multi-module"
+ThisBuild / organization := "com.rockthejvm"
+
+lazy val module_1 = (project in file("module-1"))
+lazy val module_2 = (project in file("module-2"))
+```
+Now, when we can save this file and hit _sbt_ command in the project directory. This will import the project based on the _build.sbt_ we created and will also create 2 subdirectories as _module-1_ and _module-2_ within the directory. The value provided in _file()_ is used to create the submodule name.
+
+However, as of now there is no relationship between any of the modules. We can explicitly combine both the sub modules together and link to the parent project by adding a new line as below to the _build.sbt_:
+```
+lazy val root = (project in file("."))
+ .aggregate(module_1, module_2)
+```
+
+In the same way, we can define dependencies between multiple submodules. We can make `module-2` to depend on `module-1` using `dependsOn` method:
+
+```
+lazy val module_2 = (project in file("module-2")).dependsOn(module_1)
+```
+
+We can also provide settings for each sub module differently using the `settings` method:
+
+```
+lazy val module_2 = (project in file("module-2")).settings(
+ libraryDependencies += "org.typelevel" %% "cats-effect" % "3.3.0"
+)
+```
+
+## 10. Multi-Module Build Best Practice
+We need to first identify and get clarity on different modules. Let's assume that we are building an application that contains database access, HTTP services, utilities etc.
+Each of these can be separated as a module. Then we can combine the different parts if one is dependent on another.
+
+Some good practice principles:
+- a single `build.sbt` file with all submodule information inside.
+- common settings at the top (e.g. Scala version, org name)
+- `ThisBuild` to ensure the settings are applied to the entire project including submodules
+- variables for common library used across multiple sub-modules, applied as module `libraryDependencies`
+- same for common settings, applied as `settings`
+
+Below is a sample multi-module build.sbt:
+```
+ThisBuild / scalaVersion := "2.13.8"
+ThisBuild / version := "1.0"
+ThisBuild / organization := "com.rockthejvm"
+
+val catsVersion = "2.8.0"
+val akkaVersion = "2.6.20"
+
+lazy val core = project
+ .in(file("core"))
+ .settings(
+ libraryDependencies ++= Seq(
+ "com.typesafe" % "config" % "1.4.2",
+ "org.scalameta" %% "munit" % "0.7.29" % Test
+ )
+ )
+
+val commonDeps = Seq(
+ "ch.qos.logback" % "logback-classic" % "1.4.1"
+)
+
+lazy val compilerOptions = Seq(
+ "-unchecked",
+ "-feature",
+ "-deprecation"
+)
+
+lazy val module_1 = project
+ .in(file("module-1"))
+ .settings(
+ libraryDependencies ++= commonDeps ++ Seq(
+ //add more dependencies which are needed only for this module
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion
+ )
+ )
+ .dependsOn(core)
+
+lazy val module_2 = project
+ .in(file("module-2"))
+ .settings(
+ name := "module_2", //can be different from the file() name
+ libraryDependencies ++= Seq(
+ "org.typelevel" %% "cats-core" % catsVersion
+ //add more dependencies which are needed only for this module
+ )
+ )
+ .dependsOn(core)
+
+lazy val root = project
+ .in(file("."))
+ .settings(
+ name := "multi_module",
+ publish / skip := true
+ )
+ .settings(
+ scalacOptions ++= compilerOptions ++ Seq(
+ "-Xfatal-warnings"
+ )
+ )
+ .enablePlugins(BuildInfoPlugin) //Need to add the plugin details in plugins.sbt first. In this case `sbt-buildinfo`
+ .aggregate(
+ module_1,
+ module_2
+ )
+```
+
+## 11. Executing commands on each Module
+Now that we are ready with a multi-module project, let's see how we can execute SBT commands module-wise.
+At the root of the project, if we execute `sbt` it will start the SBT session. When we run the compile command, it will compile all the modules.
+
+Once we are inside the SBT session, we can switch to a particular sub-module using:
+```
+project module_2
+```
+Now, when we compile, only this module will get compiled. But, if this module depends on another module, SBT will compile that module as well.
+
+## 12. Plugins
+
+One of the most important features of SBT is its support for plugins. Plugins help extend SBT with custom features which can be published and shared between multiple teams.
+For handling plugins, a special file called as _plugins.sbt_ is used. This file is kept under the `project/` directory. Some common usages of plugins are:
+ - packaging plugins to create jar, exe and other executables/deliverables
+ - static code analysis plugins
+ - code generation plugins
+
+ Let's look at an example of _sbt-assembly_ plugin. This plugin helps to create an executable jar file from the SBT project.
+
+ As a first step, we need to create _plugins.sbt_ and add the plugin definition to it:
+ ```
+ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0")
+ ```
+ As next step, we need to enable the required configurations needed for this plugin to work. To run a jar, we need to provide the main class to use. We can do that by providing the `mainClass` in the settings in _build.sbt_ for required module(sometimes, we need to create jar for a sub module, sometimes for the combined project)
+
+ ```
+assembly / mainClass := Some("com.rockthejvm.Module2Main"),
+ ```
+
+ Now we can use the SBT command `assembly`. This will create the jar file under the relevant project's target folder.
+ Similar to `assembly/mainClass` there are many other configurations to customise the jar creation.
+
+ This is just one of the plugins. There are many plugins available which can help improve the development experience.
+
+## 13. Global plugins
+
+ In the previous section, we added a plugin to the project. Sometimes, we might need to add some plugins that have nothing to do with the project itself. For example, there may be some plugin to publish the apps to an internal repository. This need not be kept in the git repo, instead can be shared across all the repositories.
+ SBT allows to do this using global plugins.
+
+ Instead of keeping the _plugins.sbt_ within the project, we can create the file in the _.sbt_ path.
+ ```
+ $HOME/.sbt/1.0/plugins
+ ```
+ When SBT starts, it will load all the plugins from the global path. For example, in our previous example, We can remove the _plugins.sbt_ from the _project_ directory and keep it in the above path. On reloading the project, it will still load the plugin, but from the global path instead.
+ You can notice in the SBT startup logs with some similar logs like:
+ ```
+ [info] loading global plugins from /Users/username/.sbt/1.0/plugins
+ ```
+
+## 14. Resolvers
+ So far, all the library dependencies and plugins are downloaded from the Maven Central repository. Sometimes, we need to use other third-party repository for downloading libraries/plugins. To do that, we can provide resolvers in the _.sbt_ files.
+ ```
+resolvers += Resolver.url("my-test-repo", url("https://example.org/repo-releases/"))
+ ```
+Now, apart from Maven Central, SBT will also look at the provided location for libraries.
+We can also add resolvers as the local maven directory. For that, we can use the resolver setting as:
+ ```
+resolvers += Resolver.mavenLocal
+```
+This will look for the dependencies in the `.m2` directory.
+
+Apart from Maven Central, we can also configure other services to resolve the libraries. For example, to use a Sonatype repo, we can use:
+```
+resolvers += Resolver.sonatypeRepo("releases")
+```
+We can also provide custom locations for the resolvers. For example, we can configure a customised central location for the repositories as:
+```
+resolvers += Resolver.url("my-company-repo", url("https://mycompany.org/repo-releases/"))
+```
+
+Similarly, we can add any number of resolvers.
+But note that as the number of resolvers increases, SBT might take more time to start as it might need to look at all the configured repositories before failing.
+
+## 15. Custom Tasks
+Another powerful feature of SBT is the ability to create custom tasks. Apart from the built-in task keys, we can easily create new ones. For example, we can create a custom task to do some particular operation.
+
+For example, let's create a task which will just print some text to console. We can extend this to any complex functionality in the same way.
+
+For that, we can create a Scala file which does the printing logic and keep this file under _project_ directory:
+```
+object CustomTaskPrinter {
+ def print() = {
+ println("Rock the SBT custom task is here....")
+ }
+}
+
+```
+Next, we can define a custom task in the _build.sbt_ file as:
+```
+lazy val printerTask = taskKey[Unit]("Simple custom task")
+```
+The code `taskKey[Unit]` mentions that the task does an action and just returns with a Unit type.
+
+The custom command will be `printerTask`.
+Now, we can define the logic of the custom task in the _build.sbt_ again:
+```
+printerTask := {
+ CustomTaskPrinter.print()
+}
+```
+
+After this, we can reload SBT and execute the command `printerTask`. This will print the simple message we created before into the console.
+
+Now, let's create another task to generate a string uuid value:
+
+```
+lazy val uuidStringTask = taskKey[String]("Generate string uuid task")
+uuidStringTask := {
+ StringTask.strTask()
+}
+```
+We will implement the StringTask as below:
+```
+object StringTask {
+ def strTask(): String = {
+ UUID.randomUUID().toString()
+ }
+}
+```
+
+Now we can use this task within previously created `printerTask` and access the UUID generated in the other task. Let's modify the printerTask as
+```
+printerTask := {
+ val uuid = uuidStringTask.value
+ println("Generated uuid is : "+uuid)
+ CustomTaskPrinter.print()
+}
+```
+Note that, we used `uuidStringTas.value`. This value method takes the value from the setting and assign to the val uuid.
+
+Now, when we run the command `printerTask`, it will first generate the UUID and then execute printerTask.
+
+So far, we have created custom tasks. Now, let's look at custom settings. Before that, let's understand the difference between a task and a setting.
+An SBT task is something which can be invoked and executed each time. You can think of it like `def` in Scala.
+Whereas, a setting is evaluated at the start of SBT session and after that it is memoized. It is like `val` in Scala.
+In the similar way, we can create a setting. The only difference is that instead of `taskKey` we will use `settingKey` to define a setting:
+```
+lazy val uuidStringSetting = settingKey[String]("Generate string uuid task")
+uuidStringSetting := {
+ val uuid = StringTask.strTask()
+ println("Evaluating settings... "+uuid)
+ uuid
+}
+```
+Now, to see the difference we can modify our previous printertask as:
+```
+printerTask := {
+ val uuid = uuidStringTask.value
+ println("Generated uuid from task:"+uuid)
+ val uuidSetting = uuidStringSetting.value
+ println("Generated uuid from setting:"+uuidSetting)
+ CustomTaskPrinter.print()
+}
+```
+
+Now, when we execute `printerTask`, notice the generated uuid. From task, each time a new UUID value is generated. But from setting, it will print the same value each time. Also, when you start the SBT session, immediately we can see the print statement `Evaluating settings...` with the same uuid.
+
+## 16. Command Alias
+Another advaced feature SBT supports is the ability to set aliases. This is similar to the alias we create on unix based OSs.
+
+```
+addCommandAlias("ci", ";clean; compile;test; assembly;")
+```
+
+Now, in SBT console, we can execute just `ci`. This will automatically execute the configured commands in the order it is defined in the alias _ci_
+
+## 17. Giter Templates
+SBT supports quick project bootstrap using giter(g8) templates. We can just execute the command `sbt new ` to create a new project based on the template.
+By default, we can only use the templates available under the official g8 repo. However, we can also point to any custom g8 GitHub path to create the project from that template.
+
+## 18. Cross Build between different Scala versions
+Scala releases are not binary compatible with each other(except the Scala 3 series). That means, we need to rebuild a library in all the supported versions for the users to use it.
+Doing this manually is not an easy step. SBT tries to make this easier by providing a feature to cross build between different versions.
+To support multiple versions, we can provide the settings as:
+```
+val scala212 = "2.12.16"
+val scala213 = "2.13.5"
+ThisBuild / scalaVersion := scala212
+lazy val crossproject = (project in file("crossproject"))
+ .settings(
+ crossScalaVersions := List(scala212, scala213),
+ // other settings
+ )
+```
+
+Note that, we have mentioned the default version for this project as Scala 2.12 using `ThisBuild/scalaVersion`. So when we compile, it will use Scala 2.12. Since we have given the `crossScalaVersions`, we can ask SBT to compile in all supported versions.
+To do that, we need to add the symbol `+` before SBT command:
+```
++ compile
+```
+When we publish our library, we can use `+ publish` command. This will ensure that all the supported versions of the library are published.
+
+## 19. SBT Advanced Configurations and Options
+SBT has many configuration options and argument passing on startup. Let's look at some of the important ones.
+
+### 19.1. Check SBT Version
+To check which version of SBT we are using, we can run the command:
+```
+sbt --version
+```
+
+### 19.2. View Applied SBT Options
+We can view all the applied options to SBT using the debug command.
+```
+sbt --debug
+```
+This will show a list of arguments and options applied to SBT on startup. This is very useful in identifying if the correct parameters are applied or not.
+
+Or we can use `sbt -v` which will also provide more information related to passed arguments, but not all debug logs.
+
+### 19.3. Passing Arguments on SBT Start
+We can provide jvm arguments on SBT startup. For example, if we want to increase the heap memory of the SBT process, we can start the SBT with the parameter:
+```
+sbt -v -J-Xmx3600m
+```
+### 19.4. SBT Command Alias With JVM Arguments
+We have seen before how to create SBT command aliases. We can create alias with passing jvm arguments.
+However, we need to make sure that `fork := true` is set in _build.sbt_. This setting will ensure that SBT will start a forked jvm and apply the settings.
+Without a fork JVM, the jvm parameters we pass are not considered by the SBT.
+
+Let's see how to do that.
+Let's add the following line to _build.sbt_ after adding fork:
+```
+addCommandAlias(
+ "runSpecial",
+ "; set ThisBuild/javaOptions += \"-Dport=4567\"; run"
+)
+```
+Then, we will update the Main class to read this property and print it:
+```
+val portValue = Option(System.getProperty("port"))
+println("PORT value from argument is: "+portValue)
+```
+
+Next, we need to restart SBT to get these changes to effect. Then run the command `runSpecial`. This will print the passed javaOptions.
+
+If we want to pass multiple javaOptions, we need to use a Seq as:
+```scala
+addCommandAlias(
+ "runSpecial2",
+ "; set ThisBuild/javaOptions ++= Seq(\"-Dport=4567\", \"-Duser=rockthejvm\"); run"
+)
+```
+
+Now when we run `runSpecial`, the value for `user` will be None. But when we run `runSpecial2`, it will have the value for `user` as "rockthejvm".
+
+## 20. Conclusion
+In this blog, we looked at SBT and its different components and functionalities. SBT has still more features which we haven't covered here. However, it is easy to explore more features once the basics are understood.
diff --git a/_posts/2022-10-31-sudoku-backtracking.md b/_posts/2022-10-31-sudoku-backtracking.md
new file mode 100644
index 000000000000..6add64b7f058
--- /dev/null
+++ b/_posts/2022-10-31-sudoku-backtracking.md
@@ -0,0 +1,428 @@
+---
+title: "A Backtracking Sudoku Solver in Scala"
+date: 2022-10-31
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, algorithms]
+excerpt: "How to approach Sudoku and other constraint-satisfaction problems with recursive backtracking in Scala."
+---
+
+This article is for Scala beginners. After you learn the language, the next big thing you need to master is how to write essential "algorithms" in Scala. As you can probably tell and as senior developers will no doubt tell you, the mindset that comes with the Scala language is very different from what we're usually taught in school, or in programming tutorials. This tends to make algorithms in Scala quite difficult.
+
+This article will show you how to _think_ such "algorithmic" problems with recursion, with a famous example: a Sudoku solver in Scala.
+
+> If you want to master this mindset for Google-style algorithms questions, check out the [Scala & FP interview practice course](https://rockthejvm.com/p/scala-functional-programming-interview-practice). It's a long-form, 15-hour course that will take you through all the essential data structures and algorithms, taken in a functional style with Scala.
+
+If you want to follow the video version, check out the video below or on [YouTube](https://youtu.be/zBLCbqycVzw).
+
+{% include video id="zBLCbqycVzw" provider="youtube" %}
+
+This article works identically for Scala 2 and Scala 3. In fact, the techniques are applicable to all other languages, including Java, Python, Kotlin and others. All you need is recursion.
+
+## 1. Introduction
+
+The Sudoku problem is famous, but let's state it here really quick:
+
+You have a 9x9 board, partially filled with numbers 1-9.
+You need to fill the board with numbers 1-9, such that
+ - every row
+ - column
+ - every 3x3 "box" that the big board is composed of (9 boxes in total)
+
+must contain all numbers 1-9, exactly once. Let's call this property "the Sudoku property".
+
+An example Sudoku puzzle looks like this:
+
+```text
++-------+-------+-------+
+| 5 3 _ | _ 7 _ | _ _ _ |
+| 6 _ _ | 1 9 5 | _ _ _ |
+| _ 9 8 | _ _ _ | _ 6 _ |
++-------+-------+-------+
+| 8 _ _ | _ 6 _ | _ _ 3 |
+| 4 _ _ | 8 _ 3 | _ _ 1 |
+| 7 _ _ | _ 2 _ | _ _ 6 |
++-------+-------+-------+
+| _ 6 _ | _ _ _ | 2 8 _ |
+| _ _ _ | 4 1 9 | _ _ 5 |
+| _ _ _ | _ 8 _ | _ 7 9 |
++-------+-------+-------+
+```
+
+Notice that I've visually separated the 3x3 "boxes" so that they're easier to spot - not _all_ possible 3x3 boxes inside this matrix need to have the Sudoku property.
+
+## 2. Preparation - Printing a Sudoku Game in Scala
+
+How should we represent a Sudoku puzzle in Scala? It's a 9x9 matrix, so it's going to be an array of arrays, where we're going to denote every "space" as a zero:
+
+```scala
+val problem =
+ Array(
+ Array(5,3,0, 0,7,0, 0,0,0),
+ Array(6,0,0, 1,9,5, 0,0,0),
+ Array(0,9,8, 0,0,0, 0,6,0),
+ Array(8,0,0, 0,6,0, 0,0,3),
+ Array(4,0,0, 8,0,3, 0,0,1),
+ Array(7,0,0, 0,2,0, 0,0,6),
+ Array(0,6,0, 0,0,0, 2,8,0),
+ Array(0,0,0, 4,1,9, 0,0,5),
+ Array(0,0,0, 0,8,0, 0,7,9),
+ )
+```
+
+Let's write a function that will print this puzzle nicely to the console, so that we can keep track of our attempt at filling it.
+
+```scala
+type Board = Array[Array[Int]]
+
+def prettyString(sudoku: Board): String = sudoku.mkString("\n")
+```
+
+First, we need to lay out all the rows, one by one, with a newline `\n` in between them. However, at the moment we get this:
+
+```text
+[I@467aecef
+[I@2173f6d9
+[I@307f6b8c
+[I@7a187f14
+[I@6f195bc3
+[I@51e2adc7
+[I@1a8a8f7c
+[I@2353b3e6
+[I@631330c
+```
+
+We need to unwrap the arrays as well, and turn each into a string:
+
+```scala
+def prettyString(sudoku: Board): String = sudoku.map(row => row.mkString(" ")).mkString("\n")
+```
+
+which now prints something a little bit nicer:
+
+```text
+5 3 0 0 7 0 0 0 0
+6 0 0 1 9 5 0 0 0
+0 9 8 0 0 0 0 6 0
+8 0 0 0 6 0 0 0 3
+4 0 0 8 0 3 0 0 1
+7 0 0 0 2 0 0 0 6
+0 6 0 0 0 0 2 8 0
+0 0 0 4 1 9 0 0 5
+0 0 0 0 8 0 0 7 9
+```
+
+Still quite hard to read, as we need to squint in this 9x9 matrix to find out which cell belongs to which 3x3 box, so we need to split the lines and the columns in groups of 3, each:
+
+```scala
+sudoku.grouped(3).map { bigGroup =>
+ bigGroup.map { row =>
+ row.grouped(3).map { smallGroup =>
+ smallGroup.mkString(" ")
+ }.mkString("|")
+ }.mkString("\n")
+}.mkString("\n+------------------+\n")
+```
+
+which looks a bit better:
+
+```text
+5 3 0|0 7 0|0 0 0
+6 0 0|1 9 5|0 0 0
+0 9 8|0 0 0|0 6 0
++------------------+
+8 0 0|0 6 0|0 0 3
+4 0 0|8 0 3|0 0 1
+7 0 0|0 2 0|0 0 6
++------------------+
+0 6 0|0 0 0|2 8 0
+0 0 0|4 1 9|0 0 5
+0 0 0|0 8 0|0 7 9
+```
+
+Just add a bit of padding and alignment and we should be good with the pretty printer:
+
+```scala
+ def prettyString(sudoku: Board): String = {
+ sudoku.grouped(3).map { bigGroup =>
+ bigGroup.map { row =>
+ row.grouped(3).map { smallGroup =>
+ smallGroup.mkString(" ", " ", " ")
+ }.mkString("|", "|", "|")
+ }.mkString("\n")
+ }.mkString("+-------+-------+-------+\n", "\n+-------+-------+-------+\n", "\n+-------+-------+-------+")
+ }
+```
+
+```text
++-------+-------+-------+
+| 5 3 0 | 0 7 0 | 0 0 0 |
+| 6 0 0 | 1 9 5 | 0 0 0 |
+| 0 9 8 | 0 0 0 | 0 6 0 |
++-------+-------+-------+
+| 8 0 0 | 0 6 0 | 0 0 3 |
+| 4 0 0 | 8 0 3 | 0 0 1 |
+| 7 0 0 | 0 2 0 | 0 0 6 |
++-------+-------+-------+
+| 0 6 0 | 0 0 0 | 2 8 0 |
+| 0 0 0 | 4 1 9 | 0 0 5 |
+| 0 0 0 | 0 8 0 | 0 7 9 |
++-------+-------+-------+
+```
+
+That's more like it - we can now identify things in the Sudoku matrix easily.
+
+## 3. Writing a Sudoku Validation Function in Scala
+
+The most important aspect of a constraint satisfaction problem is how to maintain the constraints _as we're building the solution_. We did something similar in the [N-Queens](/n-queens) problem. The way we're going to build up our solution will be:
+
+- on every row
+ - on every column
+ - check if there is a non-zero value there; if it is, that's part of the original puzzle
+ - if there's a zero, we'll try all numbers from 1-9:
+ - set a value
+ - _check if it satisfies the Sudoku property_
+ - move to the next cell to the right, try out all possible solutions from there
+ - unset the value and try the next one
+
+Of course, this is an imperative style, "algorithmic" approach - we'll build up a recursive solution in Scala for Sudoku shortly, but first, we need to be able to validate whether a value we're placing in an empty cell is "correct", i.e. satisfies the Sudoku property. Of course, we don't need to check _all_ rows, columns and boxes, but just those for the cell we're filling.
+
+If we're placing the number 4 at row 2, column 5, we need to check the Sudoku property for row 2, for column 5, and for the 3x3 box at "big row" 0, "big column" 1. Let's write a function to validate the property, given a Sudoku board, a row, a column and a value:
+
+```scala
+def validate(sudoku: Board, x: Int, y: Int, value: Int): Boolean = ???
+```
+
+First, let's validate the row. _Be careful_ — the row index is the coordinate y (vertical):
+
+```scala
+val row = sudoku(y)
+val rowProperty = !row.contains(value)
+```
+
+If the row already contains the value we're trying to fill, the value is invalid for this board configuration.
+
+Testing the column is a bit more tricky, because the Sudoku board is an array of _rows_, so we need to pick the "x-th" element out of each row to identify the column — thankfully, we have `map` for that:
+
+```scala
+val column = sudoku.map(r => r.apply(x))
+// same property applied to column too
+val columnProperty = !column.contains(value)
+```
+
+Now, for the box. We need to be careful here. We need to find the indices of the elements in the _corresponding 3x3 box_ of the x-y pair. Here's how we're going to find them:
+
+- Imagine the Sudoku board not as a 9x9 matrix of numbers, but as a 3x3 matrix of "boxes".
+- The coordinates of a "box" in the 3x3 matrix of "boxes" are always going to be `x/3` and `y/3`.
+- The coordinates of the top-left cell in this box are going to be `3 * xBox` and `3 * yBox`.
+- The coordinates of the bottom-right cell in this box are then going to be `3 * xBox + 2` and `3 * yBox + 2`.
+
+Let's fetch all these elements:
+
+```scala
+val boxX = x / 3
+val boxY = y / 3
+val box = for {
+ yb <- (boxY * 3) until (boxY * 3 + 3) // indices for rows in THIS box
+ xb <- (boxX * 3) until (boxX * 3 + 3) // same for cols
+} yield sudoku(yb)(xb)
+
+// apply the Sudoku property for this box as a collection of numbers:
+val boxProperty = !box.contains(value)
+```
+
+Combining all the properties gives us the final validation function:
+
+```scala
+def validate(sudoku: Board, x: Int, y: Int, value: Int): Boolean = {
+ val row = sudoku(y)
+ val rowProperty = !row.contains(value)
+
+ val column = sudoku.map(r => r.apply(x))
+ val columnProperty = !column.contains(value)
+
+ val boxX = x / 3
+ val boxY = y / 3
+ val box = for {
+ yb <- (boxY * 3) until (boxY * 3 + 3) // indices for rows in THIS box
+ xb <- (boxX * 3) until (boxX * 3 + 3) // same for cols
+ } yield sudoku(yb)(xb)
+ val boxProperty = !box.contains(value)
+
+ rowProperty && columnProperty && boxProperty
+}
+```
+
+Testing a few scenarios should validate our validation function:
+
+```scala
+validate(problem, 5, 2, 4) // true
+validate(problem, 5, 2, 5) // false
+validate(problem, 5, 2, 7) // false
+```
+
+## 4. The Recursive Backtracking Sudoku Solver
+
+Now we have everything in place. We can create a solver function that tries all possible solutions _starting at certain coordinates x and y_.
+
+```scala
+def solve(sudoku: Board, x: Int = 0, y: Int = 0): Unit = ???
+```
+
+When does this function end? Two cases:
+- we reached the end of the board, i.e. `y >= 9`, which means we've filled the board and we have a solution
+- we've exhausted all possible values
+
+The second stopping condition will arise by itself, as we'll try all numbers 1-9 and this list is finite. Let's focus on the first condition:
+
+```scala
+if (y >= 9) println(prettyString(sudoku)) // final solution
+else ... // TODO
+```
+
+If we reached the end of a row, i.e. `x >= 9`, then we need to call the `solve` function recursively starting at the next row `y + 1` and the row `0`. If it just so happens that y now becomes 9, then we're done. Otherwise, the function will continue.
+
+```scala
+else if (x >= 9) solve(sudoku, 0, y + 1) // need to fill in the next row
+else ... // TODO
+```
+
+Okay, so we're inside the board now. We need to check if the board already contains a value, as we need to skip it since it was part of the original puzzle:
+
+```scala
+else if (sudoku(y)(x) > 0) solve(sudoku, x + 1, y) // need to fill in the next cell (cell to the right)
+```
+
+Otherwise, we're at a cell which doesn't have a value yet, so we have to try all possible values which satisfy the Sudoku property, and we have a function for that!
+
+```scala
+else (1 to 9).filter(value => validate(sudoku, x, y, value)).foreach { value => /* TODO */ }
+```
+
+We keep just the values that are good candidates, i.e. pass the validation function. Inside the `foreach`, then we need to
+- put the value in the matrix
+- call `solve` on the next cell
+- remove the value in the matrix to make room for the new one
+
+```scala
+....foreach { value =>
+ // fill the sudoku board with the value
+ sudoku(y)(x) = value
+ // try the next cell
+ solve(sudoku, x + 1, y)
+ // remove the value
+ sudoku(y)(x) = 0
+}
+```
+
+So the final solver looks like this:
+
+```scala
+def solve(sudoku: Board, x: Int = 0, y: Int = 0): Unit = {
+ if (y >= 9) println(prettyString(sudoku)) // final solution
+ else if (x >= 9) solve(sudoku, 0, y + 1) // need to fill in the next row
+ else if (sudoku(y)(x) > 0) solve(sudoku, x + 1, y) // need to fill in the next cell (cell to the right)
+ else (1 to 9).filter(value => validate(sudoku, x, y, value)).foreach { value =>
+ // fill the sudoku board with the value
+ sudoku(y)(x) = value
+ // try the next cell
+ solve(sudoku, x + 1, y)
+ // remove the value
+ sudoku(y)(x) = 0
+ }
+}
+```
+
+## 5. Full Code
+
+Putting all the pieces together, we get to the final code for the Sudoku solver in Scala:
+
+```scala
+
+object Sudoku {
+
+ type Board = Array[Array[Int]]
+
+ def prettyString(sudoku: Board): String = {
+ sudoku.grouped(3).map { bigGroup =>
+ bigGroup.map { row =>
+ row.grouped(3).map { smallGroup =>
+ smallGroup.mkString(" ", " ", " ")
+ }.mkString("|", "|", "|")
+ }.mkString("\n")
+ }.mkString("+-------+-------+-------+\n", "\n+-------+-------+-------+\n", "\n+-------+-------+-------+")
+ }
+
+ def validate(sudoku: Board, x: Int, y: Int, value: Int): Boolean = {
+ val row = sudoku(y)
+ val rowProperty = !row.contains(value)
+
+ val column = sudoku.map(r => r.apply(x))
+ val columnProperty = !column.contains(value)
+
+ val boxX = x / 3
+ val boxY = y / 3
+ val box = for {
+ yb <- (boxY * 3) until (boxY * 3 + 3) // indices for rows in THIS box
+ xb <- (boxX * 3) until (boxX * 3 + 3) // same for cols
+ } yield sudoku(yb)(xb)
+ val boxProperty = !box.contains(value)
+
+ rowProperty && columnProperty && boxProperty
+ }
+
+ def solve(sudoku: Board, x: Int = 0, y: Int = 0): Unit = {
+ if (y >= 9) println(prettyString(sudoku)) // final solution
+ else if (x >= 9) solve(sudoku, 0, y + 1) // need to fill in the next row
+ else if (sudoku(y)(x) > 0) solve(sudoku, x + 1, y) // need to fill in the next cell (cell to the right)
+ else (1 to 9).filter(value => validate(sudoku, x, y, value)).foreach { value =>
+ // fill the sudoku board with the value
+ sudoku(y)(x) = value
+ // try the next cell
+ solve(sudoku, x + 1, y)
+ // remove the value
+ sudoku(y)(x) = 0
+ }
+ }
+
+ def main(args: Array[String]): Unit = {
+ val problem =
+ Array(
+ Array(5,3,0, 0,7,0, 0,0,0),
+ Array(6,0,0, 1,9,5, 0,0,0),
+ Array(0,9,8, 0,0,0, 0,6,0),
+ Array(8,0,0, 0,6,0, 0,0,3),
+ Array(4,0,0, 8,0,3, 0,0,1),
+ Array(7,0,0, 0,2,0, 0,0,6),
+ Array(0,6,0, 0,0,0, 2,8,0),
+ Array(0,0,0, 4,1,9, 0,0,5),
+ Array(0,0,0, 0,8,0, 0,7,9),
+ )
+
+ println(prettyString(problem))
+ solve(problem)
+ }
+}
+```
+
+Running the application, we then get to a predictably satisfying solution:
+
+```text
++-------+-------+-------+
+| 5 3 4 | 6 7 8 | 9 1 2 |
+| 6 7 2 | 1 9 5 | 3 4 8 |
+| 1 9 8 | 3 4 2 | 5 6 7 |
++-------+-------+-------+
+| 8 5 9 | 7 6 1 | 4 2 3 |
+| 4 2 6 | 8 5 3 | 7 9 1 |
+| 7 1 3 | 9 2 4 | 8 5 6 |
++-------+-------+-------+
+| 9 6 1 | 5 3 7 | 2 8 4 |
+| 2 8 7 | 4 1 9 | 6 3 5 |
+| 3 4 5 | 2 8 6 | 1 7 9 |
++-------+-------+-------+
+```
+
+## 6. Conclusion
+
+In this article, we learned how to approach a constraint satisfaction problem with recursive backtracking in Scala. Sudoku puzzles are always fun, and now you've written one to solve _all_ possible 9x9 Sudoku games!
diff --git a/_posts/2022-11-07-ten-scala-skills.md b/_posts/2022-11-07-ten-scala-skills.md
new file mode 100644
index 000000000000..620e4a144ceb
--- /dev/null
+++ b/_posts/2022-11-07-ten-scala-skills.md
@@ -0,0 +1,140 @@
+---
+title: "Top 10 Skills (Mostly Mental Models) to Learn to Be a Scala Developer"
+date: 2022-11-07
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, tips]
+excerpt: "Learning Scala doesn't need to be hard. Here are 10 mental skills you can learn to be a good Scala developer."
+---
+
+This article is for aspiring Scala developers. As the Scala ecosystem matures and evolves, this is the best time to become a Scala developer, and in this piece you will learn the essential tools that you should master to be a good Scala software engineer.
+
+If you're considering learning Scala, or you're at the beginning of your Scala journey and you don't know where to start, this article is for you. I teach everything that you'll see in this article, in great detail and with hands-on practice, here at [Rock the JVM](https://rockthejvm.com).
+
+> Read this article to understand what you need to work with Scala. If you then want to learn these skills hands-on, check out the [Scala Essentials](https://rockthejvm.com/p/scala) and [Advanced](https://rockthejvm.com/p/advanced-scala) courses on Rock the JVM.
+
+If you'd like to watch the video form with some nice code examples instead, please enjoy:
+
+{% include video id="kVDgurLi-CA" provider="youtube" %}
+
+## The Benefits of Scala
+
+Scala offers some of the best-paid software engineering positions by programming language (all else equal). Good Scala developers think differently, and you can only obtain this mental model with lots of practice. Therefore, Scala developers tend to be quite rare, but given the enormous value of a good Scala engineer, companies are willing to pay top dollar for such skill.
+
+Scala offers an exceptional blend of object-oriented and functional programming, with familiar syntax and concise code. Due to its structure and by following FP principles, good Scala code is often very short - much more so than other languages, e.g. Java - yet readable, fully testable and maintainable. It's no joke that sometimes you can write 10 lines of Scala with the same power as 1000 lines of equivalent Java.
+
+As you learn Scala, you will find it very applicable for amazing engineering problems with high impact. FP works miracles in distributed systems, and Scala on the JVM is a strong combination. Some powerful tools were written in Scala, specifically targeting distributed systems (e.g. Akka) and big data (e.g. Apache Spark). These problems also come with their own flavor of intellectual satisfaction, which brings me to the fact...
+
+... that once you have a taste of Scala, chances are low you'll want to go back to anything else.
+
+## The Basics
+
+Scala can be a difficult language without a good foundation. I normally teach Scala to established engineers - roughly a year of general programming experience is enough - because Scala uses a lot of software engineering concepts that cannot be learned separately one by one. Even when you write your first program, e.g.
+
+```scala
+object Main {
+ def main(args: Array[String]): Unit =
+ println("Hello, Scala")
+}
+```
+
+there are _many_ concepts that make up that program. Literally every token in there has a special meaning. Python gets a bad rap sometimes, but it deserves credit for how easy it is to get started, when your first program is
+
+```python
+print("Hello, Python")
+```
+
+That said, I've personally encountered a small <5% of students learning Scala successfully as their first programming language. It's definitely possible, but the learning curve may be a bit steeper. Still, here are some general, foundational CS concepts that you will need to learn Scala quite fast:
+
+- variables, functions and basic recursion
+- custom data types in the form of structures or classes, with their own functions (i.e. methods)
+- general OO principles, such as inheritance, interfaces or "polymorphism"
+- some common sense of code organization, style and structure - some design patterns such as factory, singleton or dependency injection would go a long way
+- using third party libraries or packages
+- some generics (or equivalent) in a typed language would work wonders - folks coming from Python or JS will likely struggle here otherwise
+- essential knowledge about threads - how they work, how you'd run some custom code on a thread
+- some light knowledge of networked/distributed systems - to understand relevant questions about serialization, communication, latency or performance
+
+These topics are usually covered in CS majors, but you can definitely learn them without a CS degree.
+
+## Fundamental Skills 1 and 2
+
+If you come from a programming language of the C family (C, C++, C#, Java, JS) you'll definitely feel at home with Scala. The syntax is quite familiar and easy to pick up.
+
+**Skill 1:** One concept that will become counterintuitive very quickly is that of _immutability_: once you define a value, it's a constant. You can't change it. Therefore, in order to obtain the equivalent of "loops" or repetitions, you need to start becoming creative very quickly. That form of creativity is _recursion_. Thankfully, recursion is no rocket science, but it may take some time until it "feels" natural. In the early stages of the [Scala Essentials course](https://rockthejvm.com/p/scala) at Rock the JVM, we start practicing recursion to get accustomed to this style of thinking, which is invaluable later.
+
+**Skill 2:** Another idea is that in Scala - and in functional programming in general - we think code in terms of _expressions_, i.e. things that are evaluated, instead of _instructions_, i.e. things that are executed step by step. This is one fundamental difference between functional programming and imperative programming in terms of mental model.
+
+As you progress through this stage, a mental shift will start to take place. Writing Scala is not a big deal - _thinking_ in Scala is.
+
+## Skill 3: Object-Oriented Programming
+
+OOP principles are quite similar in Scala compared to other languages. The general ideas of
+
+- classes, constructors and instances
+- inheritance
+- traits (interfaces) and abstract classes
+- subtype polymorphism
+- methods, overloading and overriding
+- generics
+
+are mostly the same. Some Scala original innovations are
+
+- case classes, now adopted by other JVM languages like Kotlin (called data classes) and recent versions of Java (called records)
+- objects and companions implementing the equivalent of "static" fields and methods in Java
+
+and they are the bread and butter of code and data organization, so you need to be good users of them.
+
+**Skill 3:** master object-oriented programming. If you're coming to Scala from another language, chances are high you're already well versed.
+
+## Skill 4: Pattern Matching
+
+Pattern matching is one of Scala's most powerful features, eliminating vast amounts of boilerplate. Learn what pattern matching is and how it works. As you become more experienced with pattern matching, you can also learn some nicer [pattern matching tricks](https://blog.rockthejvm.com/8-pm-tricks/) to speed up your development.
+
+It's so useful and powerful, that it occurs everywhere. I've never heard of a Scala developer that doesn't use pattern matching.
+
+## Skills 5 and 6 of Functional Programming
+
+One core idea of functional programming is that you can work with functions as values: pass them around as arguments, return them as results, construct new ones on the fly - in other words, work with functions like you would with any other kinds of values. We call functions "first-class" elements of the language.
+
+Because Scala originally targeted the JVM (built for Java, an OO language without FP features), it came up with an ingenious idea: functions in Scala are actually instances of interfaces/traits called `FunctionX` (where X is 0,1,2,3...), with a method called `apply`. Apply methods in a class allow an instance of that class to be directly "invoked", as if it were a function - which is exactly what these FunctionX interfaces do. This idea blew my mind when I first encountered it.
+
+**Skill 5:** Therefore, the ability to use functions as values - pass them around, construct new ones on the spot, etc. - is a crucial skill for a functional Scala developer. Master it and you're well on your way.
+
+**Skill 6:** With functional programming, we can then start to think of collections, such as List, Set, Map, Vector, etc, and process them quickly with the help of FP-style combinators. The functions `map`, `flatMap` and `filter` are your best friends. With them, you can then build the famous for-comprehensions in Scala, which are nothing more than chains of `map` and `flatMap`. This will be another counterintuitive idea, because we programmers tend to think "for" as a "loop" (the C-style language curse), whereas here, a for-comprehension is an expression.
+
+## Skills 7 and 8 of Abstract Reasoning
+
+**Skill 7:** After collections, our newly developed FP skills will allow us to understand other structures that look like collections, but they _mean_ something else. Two quick examples are Option and Try: these are data structures that store a single value, but the abstract meaning is "potentially absent value" and "potentially failed computation", respectively. Working with them is strikingly similar to working with collections, because we have similar `map`, `flatMap`, `filter`, for comprehensions, plus other combinators that allow us to deal with failures and absence of values, without having to directly check them! These structures eliminate vast amounts of boilerplate, nested checks and reading headaches, while maintaining code correctness, which is a massive productivity boost.
+
+**Skill 8:** If you have more time, then, it's worth diving into the abstract world of (gasp) monads - the kind of computations that can be "chained". I've explained monads from 3 different angles ([pure productivity](/monads), [general properties](/another-take-on-monads) and ["monoids in the category of endofunctors"](/monads-are-monoids-in-the-category-of-endofunctors)), if you want to check those pieces. Monads seem to be a "sweet spot" in functional programming, because they enable the sort of pragmatic programming that produces useful values. Cats Effect and ZIO, two of the most powerful libraries in the Scala world, are heavily based on these concepts.
+
+## Skill 9: Functional Multithreading
+
+**Skill 9:** It's worth understanding how Futures work in Scala. Even as many Scala developers detest them, Futures are an essential tool for manipulating unfinished expressions on another thread. As with Option and Try, Futures can be processed with similar FP combinators you may have learned beforehand, which means you'll now be able to design multithreaded programs with similar code as that of a single-threaded program. This is huge.
+
+## Skill 10: Contextual Abstractions (Scala 2: Implicits)
+
+Contextual abstractions are tools that allow Scala code to be "correct" (i.e. compile) only in a particular set of circumstances (contexts). There are 3 major contextual abstractions that you should definitely know.
+
+The first is the `given`/`using` combo. Given values are automatically injected by the compiler in methods that have a `using` clause. At first, you can think about such methods as having default values, except you don't provide those default values in advance, but you create them elsewhere as a `given`, and the compiler will then pass them in your stead.
+
+The second contextual abstraction is extension methods. Scala has this great capability to add new methods to existing types. In this way, your libraries, API and regular code will become shorter, more expressive, and easier to read and understand.
+
+The third major contextual abstraction is conversions. This has taken a less important role recently in Scala 3, but it's worth knowing regardless, because otherwise you may see code that "magically" compiles, and you won't understand why.
+
+These 3 major contextual abstractions give rise to a whole new world of APIs and design patterns. A major design pattern is _type classes_, which you should learn as a Scala developer. Many tools in the Scala ecosystem - particularly the Typelevel stack - use type classes a lot.
+
+**Skill 10:** if you can understand and internalize the way contextual abstractions work, you're on your way to becoming a great Scala developer.
+
+## Bonus Skill: Honorable Mentions of the Scala Type System
+
+Scala is immensely powerful, and the features that I've outlined so far will get you 80% of the benefits of Scala as a language and tool. If you're designing your own APIs or critical/infrastructure code, it's worth exploring Scala's type system in more depth. Some of the most important things that you can learn are
+
+- Variance. Libraries like ZIO leverage this feature like no other in the Scala world.
+- Self-types. OO programmers will find this feature quite useful
+- Type members and type projections. These will help you design APIs with maximum flexibility.
+
+# Conclusion
+
+In this article, we've explored some critical skills and concepts that you can learn as a Scala developer. If you learn the above, you'll be well ahead of most folks learning Scala from bits and pieces around the internet or StackOverflow.
diff --git a/_posts/2022-11-28-finagle.md b/_posts/2022-11-28-finagle.md
new file mode 100644
index 000000000000..cba20ab0ec96
--- /dev/null
+++ b/_posts/2022-11-28-finagle.md
@@ -0,0 +1,288 @@
+---
+title: "Finagle Tutorial: Twitter's RPC Library for Scala"
+date: 2022-11-28
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, finagle]
+excerpt: "Finagle is a powerful and deceptively simple library for distributed systems, with built-in HTTP support, load balancing and more"
+---
+
+This article is for Scala developers of all levels - you don't need any fancy Scala knowledge to make the best out of this piece.
+
+For those who don't know yet, almost the entire Twitter backend runs on Scala, and the [Finagle](https://github.com/twitter/finagle) library is at the core of almost all Twitter services. They also built an entire ecosystem of libraries, tools and frameworks on top of Finagle, which have been successfully used in production at other big companies (e.g. Foursquare, ING, Pinterest, Soundcloud, etc).
+
+As I'm writing this article, Twitter is going through a major shakeup. More than half the staff have been fired, and more have resigned, including the entire Finagle team. However, the success of Twitter as a distributed system is _because_ of Scala and Finagle; moving away from Scala/Finagle would be a dumb and costly move.
+
+I expect this article to still be relevant, even just for you to understand some solid principles which you can use to build your applications, regardless of library or tool, even if Finagle ceases maintenance.
+
+If you'd like to watch the video form:
+
+{% include video id="EbWuiURnScw" provider="youtube" %}
+
+## 1. Finagle Services
+
+In order to add Finagle to a `build.sbt` project we'll need these definitions:
+
+```scala
+libraryDependencies ++= Seq(
+ "com.twitter" %% "finagle-core" % "22.7.0",
+ "com.twitter" %% "finagle-http" % "22.7.0",
+)
+```
+
+As of this writing, Finagle supports Scala 2.13. As Finagle (to my knowledge) doesn't use any Scala fancy tools (e.g. macros), supporting Scala 3 shouldn't be difficult to add.
+
+Finagle is an _RPC_ library for distributed systems. This means it's protocol-agnostic: we can interact with another machine via any sort of protocol: gRPC, HTTP, Thrift, etc. In this article we'll demonstrate the principles by means of HTTP.
+
+RPC is an old term for "remote procedure call", which in our case is a bit outdated, but the principle still stands. A Finagle "service" is described by a function
+
+```scala
+abstract class Service[-A, +B] extends (A => Future[B])
+```
+
+That's it. A Service is a function that takes in an argument and returns a result _later_. Normally we think of such functions as async functions on the same machine, but Finagle extends this concept to include distributed systems: our result may be (and often is) returned from _another system_. This means that the protocol to interact with the outside system, as well as the serialization/deserialization between our "native" values and the data the protocol understands, are all abstracted away.
+
+This is the single most powerful concept in Finagle.
+
+A note that the `Future` type we use in this signature is Twitter's own `Future` type, not that from `scala.concurrent`. It has the same semantics and similar transformers (e.g. `map`, `flatMap`, for-comprehensions, `onComplete`) as the built-in one. If you're wondering why Twitter isn't using the `scala.concurrent` version, it's because Twitter's Future is older than Scala's Future. It's no exaggeration that some Twitter packages inspired what's now in the Scala standard library.
+
+## 2. HTTP Services and Clients
+
+Let's demonstrate some Finagle services with HTTP. Let's consider a server that listens on port 9090 and receives requests at URLs of the form `localhost:9090?name=daniel` and returns HTTP responses with the length of the query parameter "name", as the payload.
+
+```scala
+import com.twitter.finagle._
+import com.twitter.finagle.http._
+import com.twitter.util._
+```
+
+```scala
+def stringLengthService = new Service[Request, Response] {
+ override def apply(request: Request): Future[Response] = Future {
+ val computationResult = Option(request.getParam("name")).map(_.length).getOrElse(-1)
+ val response = Response(Status.Ok)
+ response.setContentString(computationResult.toString)
+ response
+ }
+}
+```
+
+An HTTP service is a Service that takes in HTTP `Request`s and returns Futures holding HTTP `Response`s. This particular service parses the request, takes its query parameter value and computes its length, then builds a Response (note it's mutable) and returns it.
+
+The main application can simply say
+
+```scala
+def main(args: Array[String]): Unit = {
+ val server = Http.serve(":9090", stringLengthService)
+ Await.ready(server)
+}
+```
+
+At this point, we can start our application an issue HTTP requests via cURL or HTTPie or your favorite HTTP tool (even your browser).
+
+```shell
+$ http get localhost:9090?name=daniel
+
+HTTP/1.1 200 OK
+Content-Length: 1
+
+6
+```
+
+We can also build clients that interact with this HTTP server. In another Scala object, we can build one:
+
+```scala
+object SimpleClient {
+ def main(args: Array[String]): Unit = {
+ val client: Service[Request, Response] = Http.newService("localhost:9090")
+
+ val request = Request(Method.Get, "/?name=daniel")
+ val response: Future[Response] = filteredClient(request)
+
+ response.onSuccess(resp => println(resp.getContentString()))
+ response.onFailure(ex => ex.printStackTrace())
+ }
+}
+```
+
+Through `Http.newService(...)` we abstract away a Service, which we can now invoke like a function, _locally_, while the response is actually fetched from the other system. This is what makes Finagle an RPC system. Running this `SimpleClient` application will print the same value as the shell example.
+
+Also note how Futures can be handled by supplying callback _lambdas_, much like we now do with the Scala standard Futures.
+
+## 3. Filters
+
+Services can be easily built and used, as shown in the previous section. Often, though, we would like to build extra layers of logic on top of existing services. Examples:
+
+- Data transformations between the inputs/outputs that the service supports and our desired data structures. Example: encoders/decoders
+- Extra conditions on the inputs, outputs or other attributes of the service itself. Example: enforcing SLAs or input sanitizing
+
+These extra layers of logic are sometimes called "middleware" in other library ecosystems (e.g. Typelevel or ZIO), here they're called Filters.
+
+A Filter has 4 type arguments, following the flow of data in this gorgeous ASCII diagram I made myself:
+
+```text
+ ReqIn +--------+ RepOut +---------+
+ ------> | | -------> | |
+ | Filter | | Service |
+ <------ | | <------- | |
+ RepIn +--------+ ReqOut +---------+
+```
+
+A Filter is placed "in front of" the service we would like to enhance. Thinking of function composition, this diagram has 3 functions:
+
+- a `ReqIn => RepOut` function which transforms the input into something that the service understands
+- the `RepOut => ReqOut` function which is the service itself (remember that Service extends Function1); technically this function is `RepOut => Future[ReqOut]`
+- the `ReqOut => RepIn` function which transforms the output of the service into another type for the outside world
+
+Compacting these 3 functions into one, this leads to a big function `(ReqIn, (RepOut => Future[ReqOut])) => Future[RepIn]` or `(ReqIn, Service[RepOut,ReqOut]) => Future[RepIn]` which is the fundamental description of a Filter:
+
+```scala
+abstract class Filter[-ReqIn, +RepOut, +ReqOut, -RepIn]
+ extends ((ReqIn, Service[ReqOut, RepIn]) => Future[RepOut])
+```
+
+Because many filters don't need data transformation, we also have SimpleFilters which do no data transformations
+
+```scala
+abstract class SimpleFilter[Req, Rep] extends Filter[Req, Rep, Req, Rep]
+```
+
+With this new concept, let's add a new filter which enforces replies to be sent back within one second, otherwise the Future holding the reply will be failed:
+
+```scala
+class TimeoutFilter[Req, Rep](timeout: Duration, timer: Timer) extends SimpleFilter[Req, Rep] {
+ override def apply(request: Req, service: Service[Req, Rep]): Future[Rep] =
+ service(request).within(timer, timeout)
+}
+```
+
+With this filter implemented, adding it is easy once we have an existing service. We can add it in front of our HTTP client to enforce SLAs, for example:
+
+```scala
+object TimeoutClient {
+ def main(args: Array[String]): Unit = {
+ val originalClient: Service[Request, Response] = Http.newService("localhost:9090")
+ val timeoutFilter = new TimeoutFilter[Request, Response](Duration.fromSeconds(1), new JavaTimer())
+ val filteredClient = timeoutFilter.andThen(originalClient)
+
+ val request = Request(Method.Get, "/?name=daniel")
+ val response: Future[Response] = filteredClient(request)
+ // asynchronous API
+ // map, flatMap, for comprehensions
+ response.onSuccess(resp => println(resp.getContentString()))
+ response.onFailure(ex => ex.printStackTrace())
+ Thread.sleep(2000)
+ }
+}
+```
+
+The critical line is `val filteredClient = timeoutFilter.andThen(originalClient)`, which is an intuitive API. This also makes it easy for us to place multiple filters before a service, each time returning a new service.
+
+This application will work as before, since the HTTP server's Futures are completed immediately. If we add a `Thread.sleep(1200)` inside the HTTP server's implementation, for example:
+
+```scala
+def stringLengthService = new Service[Request, Response] {
+ override def apply(request: Request): Future[Response] = Future {
+ val computationResult = Option(request.getParam("name")).map(_.length).getOrElse(-1)
+ val response = Response(Status.Ok)
+ response.setContentString(computationResult.toString)
+ Thread.sleep(1200) // simulate taking a long time
+ response
+ }
+}
+```
+
+then if we restart the server, our client will now see a failed Future, and we'll see a timeout exception with a stack trace.
+
+## 4. Load Balancing
+
+This feature of Finagle is quite spectacular. Load balancing is important in distributed systems, and if we have multiple instances of the same application hosted on many machines, it's often useful to distribute the load among them. Finagle makes this easy.
+
+Finagle has a `Name` data structure which can be bound to a client. For single-instance services, the Name contains a single address. But we can instantiate multiple HTTP services on multiple ports to demonstrate how Names can be bound to all these instances at once.
+
+First, to modify the HTTP service code a bit:
+
+```scala
+def debugFilter(id: String) = new SimpleFilter[Request, Response] {
+ override def apply(request: Request, service: Service[Request, Response]) = {
+ println(s"[${id}] received request $request")
+ service(request)
+ }
+}
+
+// same stringLengthService implementation
+
+def simpleHttpServer(port: Int) =
+ Http.serve(s":${port}", debugFilter(s"server-$port").andThen(stringLengthService))
+
+def main(args: Array[String]): Unit = {
+ (9090 to 9090).map { port =>
+ simpleHttpServer(port)
+ }.foreach(server => Await.ready(server))
+}
+```
+
+This code spins up 3 different HTTP servers on the same JVM. We can of course start 3 separate instances of the same HTTP server on different JVMs, but this one is easier to manage and the result is the same.
+
+We first add a debugger in the form of the `debugFilter` method, which returns a Filter — notice how useful the concept is — which prints the request along with the server id, so that we can track which request arrived at which server, from the same console.
+
+We also changed how the HTTP service is served with the `simpleHttpServer` method, then started 3 servers on ports 9090, 9091 and 9092.
+
+From the point of view of the client now, things are comparatively simpler (as they should):
+
+```scala
+object LoadBalancedClient {
+ def main(args: Array[String]): Unit = {
+ val addresses = (9090 to 9092).toList.map(port => Address("localhost", port))
+ val name: Name = Name.bound(addresses: _*)
+ val client = Http.newService(name, "client")
+ val requests = (1 to 20).map(i => Request(Method.Get, s"?name=${"daniel" * i}"))
+ val responses = requests.map(req => client(req).map(_.getContentString))
+ // collect = "traverse"
+ Future.collect(responses).foreach(println)
+ Thread.sleep(5000)
+ }
+}
+```
+
+After building the addresses, we bind them to the Name data structure, and then the Name in turn to `val client = Http.newService(name, "client")`. After this point, the client can be used _exactly_ as before, as a function `Request => Future[Response]`! Except in this case, the requests will be (internally) redirected to the 3 HTTP servers. The algorithm for routing is outside the scope of this article, but it can be configured, and you can read more about the [built-in](https://twitter.github.io/finagle/guide/Clients.html#power-of-two-choices-p2c-least-loaded) [algorithms](https://twitter.github.io/finagle/guide/ApertureLoadBalancers.html) in the docs.
+
+After load-balancing, the API stays consistent, and the client doesn't have (and doesn't need to have) any knowledge about how the service is implemented, or even how many instances of the service there are.
+
+Running the application, we'll see that the requests are cycled in between server instances:
+
+```shell
+[server-9090] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58174)
+[server-9092] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58171)
+[server-9091] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58176)
+[server-9092] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58169)
+[server-9090] received request Request("GET ?name=danieldanieldanieldanieldanieldaniel", from /127.0.0.1:58177)
+[server-9091] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58181)
+[server-9092] received request Request("GET ?name=danieldanieldanieldanieldaniel", from /127.0.0.1:58185)
+[server-9090] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58175)
+[server-9090] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58168)
+[server-9092] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58173)
+[server-9092] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58180)
+[server-9092] received request Request("GET ?name=danieldanieldaniel", from /127.0.0.1:58179)
+[server-9092] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58183)
+[server-9091] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58166)
+[server-9090] received request Request("GET ?name=danieldaniel", from /127.0.0.1:58178)
+[server-9090] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58182)
+[server-9091] received request Request("GET ?name=danieldanieldanieldaniel", from /127.0.0.1:58184)
+[server-9091] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58170)
+[server-9091] received request Request("GET ?name=daniel", from /127.0.0.1:58172)
+[server-9090] received request Request("GET ?name=danieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldanieldaniel", from /127.0.0.1:58167)
+```
+
+This feature makes it very easy for us to spin up additional instances of a service: just start it, and add it to a configuration file (or something persistent) that a client can now use, and the client will start directing traffic towards the new instance as well.
+
+## 5. Conclusion
+
+In this article, we've looked at the basics of the Finagle RPC library:
+
+- how services are described and consumed
+- how to build HTTP servers and clients
+- how to add filters and "middleware" on top of existing services
+- how to add load balancing while keeping the consumer API intact
+
+The Finagle ecosystem of libraries is extremely powerful. I personally doubt they will cease development, given the success they've had with massively distributed systems (not just at Twitter). If nothing else, we should all learn from Finagle's philosophy for **our own tools and libraries, and I hope this article helped achieve that goal.
diff --git a/_posts/2023-01-03-kotlin-coroutines-101.md b/_posts/2023-01-03-kotlin-coroutines-101.md
new file mode 100644
index 000000000000..e66cb9c1223e
--- /dev/null
+++ b/_posts/2023-01-03-kotlin-coroutines-101.md
@@ -0,0 +1,929 @@
+---
+title: "Kotlin Coroutines - A Comprehensive Introduction"
+date: 2023-01-03
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kotlin, coroutines, concurrency]
+excerpt: "Kotlin coroutines are a powerful tool for asynchronous programming, and they fall under the umbrella of structured concurrency. Let's check out their main features and strengths in this tutorial."
+---
+
+_This article is brought to you by [Riccardo Cardin](https://github.com/rcardin). Riccardo is a proud alumnus of Rock the JVM, now a senior engineer working on critical systems written in Scala and Kotlin._
+
+_Enter Riccardo:_
+
+This article introduces Kotlin coroutines, a powerful tool for asynchronous programming. Kotlin's coroutines fall under the umbrella of structured concurrency. They implement a model of concurrency which you can consider similar to Java virtual threads, [Cats Effect](https://blog.rockthejvm.com/cats-effect-fibers/) and [ZIO fibers](https://blog.rockthejvm.com/zio-fibers/). In detail, we'll present some use cases concerning the use of coroutines on backend services, not on the Android environment.
+
+The article requires existing knowledge of Kotlin.
+
+> Coroutines can be tough. If you need to get the Kotlin fundamentals **fast** and with thousands of lines of code and a project under your belt, you'll love [Kotlin Essentials](https://rockthejvm.com/p/kotlin-essentials). It's a jam-packed course on **everything** you'll ever need to work with Kotlin for any platform (Android, native, backend, anything), including less-known techniques and language tricks that will make your dev life easier. Check it out [here](https://rockthejvm.com/p/kotlin-essentials).
+
+## 1. Background and Setup
+
+All the examples we'll present requires at least version 1.7.20 of the Kotlin compiler and version 1.6.4 of the Kotlin Coroutines library. The basic building blocks of coroutines are available in the standard library. The full implementation of the structured concurrency model is in an extension library called `kotlinx-coroutines-core`.
+
+We'll use a Maven file to resolve dependency and build the code. We shared an example of `pom.xml` file at the end of this article. It's also possible to create a similar building file for Gradle, but we'll stick to Maven for simplicity.
+
+During the article, we'll use an Slf4j logger to print the output of the code instead of using the `println` function:
+
+```kotlin
+val logger: Logger = LoggerFactory.getLogger("CoroutinesPlayground")
+```
+
+The logger allows us to easily trace the id of the coroutine running the code and the thread's name executing it. Remember: To see the identifier of the coroutine, we need to add the following VM property when running the code:
+
+```
+-Dkotlinx.coroutines.debug
+```
+
+For example, with the above setup, we will have the following output:
+
+```text
+14:59:20.741 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Boiling water
+```
+
+In the above example, the `DefaultDispatcher-worker-1` represents the thread's name. The `coroutine` string is the default coroutine name, whereas the `#2` string represents the identifier.
+
+## 2. Why Coroutines?
+
+The first question that comes to mind is: why should we use coroutines? The answer is simple: coroutines are a powerful tool for asynchronous programming. Someone could argue that we already have the `Thread` abstraction in the JVM ecosystem that models an asynchronous computation. However, **threads that the JVM maps directly on OS threads are heavy**. For every thread, the OS must allocate a lot of context information on the stack. Moreover, every time a computation reaches a blocking operation, the underneath thread is paused, and the JVM must load the context of another thread. The context switch is costly, so we should avoid blocking operations in our code.
+
+On the other end, as we will see, **coroutines are very lightweight**. They are not mapped directly on OS threads but at the user level, with simple objects called _continuations_. Switching between coroutines does not require the OS to load another thread's context but to switch the reference to the continuation object.
+
+Another good reason to adopt **coroutines** is that they **are a way to write asynchronous code in a synchronous fashion**.
+
+As an alternative, we can use callbacks. However, callbacks are not very elegant, and they are not composable. Moreover, it's not very easy to reason about them. It's easy to end up in a *callback hell*, where the code is tough to read and maintain:
+
+
+```kotlin
+a(aInput) { resultFromA ->
+ b(resultFromA) { resultFromB ->
+ c(resultFromB) { resultFromC ->
+ d(resultFromC) { resultFromD ->
+ println("A, B, C, D: $resultFromA, $resultFromB, $resultFromC, $resultFromD")
+ }
+ }
+ }
+}
+```
+
+The example above shows the execution of four functions using the callback style. As we can see, collecting the four values returned by the four functions takes a lot of work. Moreover, the code could be easier to read and maintain.
+
+Another model that is used in asynchronous programming is reactive programming. However, the problem is that it needs to produce more complex code to understand and maintain. Let's take, for example, the following code snippet from the official documentation of the RxJava library:
+
+```java
+Flowable.fromCallable(() -> {
+ Thread.sleep(1000); // imitate expensive computation
+ return "Done";
+})
+ .subscribeOn(Schedulers.io())
+ .observeOn(Schedulers.single())
+ .subscribe(System.out::println, Throwable::printStackTrace);
+```
+
+The above code simulates the run of some computation, and network request on a background thread, showing the results (or error) on the UI thread. It's not self-explanatory, and we need to know the library well to understand what's happening.
+
+Coroutines solve all the above problems. Let's see how.
+
+## 3. Suspending Functions
+
+To start, you can think of a coroutine as a lightweight thread, which means it's not mapped directly to an OS thread. It's a computation that can be suspended and resumed at any time. So, before we can start looking at how to build a coroutine, we need to understand how to suspend and resume a coroutine.
+
+**Kotlin provides the `suspend` keyword to mark a function that can suspend a coroutine**, i.e. allow it to be paused & resumed later:
+
+```kotlin
+suspend fun bathTime() {
+ logger.info("Going to the bathroom")
+ delay(500L)
+ logger.info("Exiting the bathroom")
+}
+```
+
+If you're a Scala geek and have been following us for a while, you may notice the example is the same as the [ZIO Fibers article](https://blog.rockthejvm.com/zio-fibers/) - a great opportunity for you to see how coroutines are different from fibers.
+
+The `delay(timeMillis: Long)` function is a `suspend` that suspends a coroutine for `timeMillis` milliseconds. A `suspend` function can be called only from a coroutine or another `suspend` function. It can be suspended and resumed. In the example above, the `bathTime` function can be suspended when the coroutine executes the `delay` function. Once resumed, the `bathTime` function will continue its execution from the line immediately after the suspension.
+
+The above mechanism is wholly implemented in the Kotlin runtime, but how is it possible? Without going too deep into coroutines' internals, **the whole context of the suspending function is saved in an object of type `Continuation`**. The `T` type variable represents the function's return type. The continuation contains all the state of the variables and parameters of the function. Moreover, it includes a label that stores the point where the execution was suspended. So, the Kotlin compiler will rewrite every suspending function adding a parameter of type `Continuation` to the function signature. The signature of our `bathTime` function will be rewritten as follows:
+
+```kotlin
+fun bathTime(continuation: Continuation<*>): Any
+```
+
+Why the compiler also changes the return type? The answer is that when the `suspend` function is suspended, it cannot return the value of the function. Still, it must return a value that marks that the function was suspended, `COROUTINE_SUSPENDED`.
+
+Inside the `continuation` object, the compiler will save the state of the execution of the function. Since we have no parameters and no internal variables, the continuation stores only a label marking the advance of the execution. For the sake of simplicity, let's introduce a `BathTimeContinuation` type to store the context of the function.
+
+In our example, the runtime can call the `bathTime` function at the beginning or after the `delay` function. If we use an `Int` label, we can represent the two possible states of the function as follows:
+
+```kotlin
+fun bathTime(continuation: Continuation<*>): Any {
+ val continuation =
+ continuation as? BathTimeContinuation ?: BathTimeContinuation(continuation)
+ if (continuation.label == 0) {
+ logger.info("Going to the bathroom")
+ continuation.label = 1
+ if (delay(500L, continuation) == COROUTINE_SUSPENDED) return COROUTINE_SUSPENDED
+ }
+ if (continuation.label == 1) {
+ logger.info("Exiting the bathroom")
+ }
+ error("This line should never be reached")
+}
+```
+
+Well, a lot of things are happening here. First, we must check if the `continuation` object is of type `BathTimeContinuation`. If not, we create a new `BathTimeContinuation` object, passing the `continuation` object as a parameter. We create a new continuation instance when the `bathTime` function is called for the first time. As we can see, continuations are like onions: **Every time we call a suspending function, we wrap the continuation object in a new one**.
+
+Then, if the `label` is `0`, we print the first message and set the label to `1`. Then, we call the `delay` function, passing the continuation object. If the `delay` function returns `COROUTINE_SUSPENDED`, it means that the function was suspended, and we return `COROUTINE_SUSPENDED` to the caller. Suppose the `delay` function returns a value different from `COROUTINE_SUSPENDED`. In that case, it means the function resumed, and we can continue the execution of the `bathTime` function. If the label is `1`, the function is just resumed, and we print the second message.
+
+The above is a simplified version of the actual code generated by the Kotlin compiler and run by the Kotlin runtime. Though, it's enough to understand how coroutines work.
+
+## 4. Coroutine Scope and Structural Concurrency
+
+Now we can start looking at how Kotlin implements the concept of structural concurrency. Let's declare another suspending function, which will simulate the action of boiling some water:
+
+```kotlin
+suspend fun boilingWater() {
+ logger.info("Boiling water")
+ delay(1000L)
+ logger.info("Water boiled")
+}
+```
+
+The first function we introduce is the `coroutineScope` suspending function. This function is at the core of coroutines and is used to create a new coroutine scope. It takes a suspending lambda as a parameter with an instance of `CoroutineScope` as the receiver:
+
+```kotlin
+suspend fun coroutineScope(
+ block: suspend CoroutineScope.() -> R
+): R
+```
+
+**The coroutines scope represents the implementation of structural concurrency in Kotlin**. The runtime blocks the execution of the `block` lambda until all the coroutines started inside the `block` lambda are completed. These coroutines are called children coroutines of the scope. Moreover, structural concurrency also brings us the following features:
+
+* Children coroutines inherit the context (`CoroutineContext`) of the parent coroutine, and they can override it. The coroutine's context is part of the `Continuation` object we've seen before. It contains the name of the coroutine, the dispatcher (aka, the pool of threads executing the coroutines), the exception handler, and so on.
+* When the parent coroutine is canceled, it also cancels the children coroutines.
+* When a child coroutine throws an exception, the parent coroutine is also stopped.
+
+In addition, the `coroutineScope` function also creates a new coroutine, which suspends the execution of the previous one until the end of its execution. So, if we want to execute the two steps of our morning routine sequentially, we can use the following code:
+
+```kotlin
+suspend fun sequentialMorningRoutine() {
+ coroutineScope {
+ bathTime()
+ }
+ coroutineScope {
+ boilingWater()
+ }
+}
+```
+
+To execute the `sequentialMorningRoutine`, we must declare a suspending `main` function that we'll reuse throughout the rest of the article:
+
+```kotlin
+suspend fun main() {
+ logger.info("Starting the morning routine")
+ sequentialMorningRoutine()
+ logger.info("Ending the morning routine")
+}
+```
+
+The `sequentialMorningRoutine` function will execute the `bathTime` function sequentially and then the `boilingWater` function in two different coroutines. So, we shouldn't be surprised that the output of the above code is something similar to the following:
+
+```text
+15:27:05.260 [main] INFO CoroutinesPlayground - Starting the morning routine
+15:27:05.286 [main] INFO CoroutinesPlayground - Going to the bathroom
+15:27:05.811 [kotlinx.coroutines.DefaultExecutor] INFO CoroutinesPlayground - Exiting the bathroom
+15:27:05.826 [kotlinx.coroutines.DefaultExecutor] INFO CoroutinesPlayground - Boiling water
+15:27:06.829 [kotlinx.coroutines.DefaultExecutor] INFO CoroutinesPlayground - Water boiled
+15:27:06.830 [kotlinx.coroutines.DefaultExecutor] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+As we can see, the execution is purely sequential. However, we can see that the runtime uses two different threads to execute the whole process, the `main` and the `kotlinx.coroutines.DefaultExecutor` thread. An important property of coroutines is that **when they are resumed, they can be executed in a different thread than the one that suspended them**. For example, the `bathTime` coroutine starts on the main thread. Then, the `delay` function suspends it. Finally, it is resumed on the `kotlinx.coroutines.DefaultExecutor` thread.
+
+## 5. Coroutine Builders
+
+### 5.1. The `launch` Builder
+
+At this point, we should know about suspending functions and the basics of structural concurrency. It's time to create our first coroutine explicitly. **The Kotlin coroutines library provides a set of functions called builders**. These functions are used to create a coroutine and to start its execution. The first function we'll see is the `launch` function:
+
+```kotlin
+public fun CoroutineScope.launch(
+ context: CoroutineContext = EmptyCoroutineContext,
+ start: CoroutineStart = CoroutineStart.DEFAULT,
+ block: suspend CoroutineScope.() -> Unit
+): Job
+```
+
+The library defines the `launch` builder as an extension function of the `CoroutineScope`. So, we need a scope to create a coroutine in this way. To create a coroutine, we also need a `CoroutineContext` and a lambda with the code to execute. The builder will pass its `CoroutineScope` to the `block` lambda as the receiver. This way, we can reuse the scope to create new children coroutines. Finally, the builder's default behavior is to immediately start the new coroutine (`CoroutineStart.DEFAULT`).
+
+So, let's add some concurrency to our morning routine. We can start the `boilingWater` and the `bathTime` functions in two new coroutines and see them racing:
+
+```kotlin
+suspend fun concurrentMorningRoutine() {
+ coroutineScope {
+ launch {
+ bathTime()
+ }
+ launch {
+ boilingWater()
+ }
+ }
+}
+```
+
+The log of the above code is something similar to the following:
+
+```text
+09:09:44.817 [main] INFO CoroutinesPlayground - Starting the morning routine
+09:09:44.870 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Going to the bathroom
+09:09:44.871 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Boiling water
+09:09:45.380 [DefaultDispatcher-worker-2 @coroutine#1] INFO CoroutinesPlayground - Exiting the bathroom
+09:09:45.875 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Water boiled
+09:09:45.876 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+We can extract a lot of information from the above log. First, we can see that we effectively spawned two new coroutines, `coroutine#1` and `coroutine#2`. The first runs the `bathTime` suspending function, and the second the `boilingWater`.
+
+The logs of the two functions interleave, so the execution of the two functions is concurrent. **This model of concurrency is cooperative** (more in the following sections). The `coroutine#2` had a chance to execute only when `coroutine#1` reached the execution of a suspending function, i.e., the `delay` function.
+
+Moreover, when suspended, the `coroutine#1` was running on thread `DefaultDispatcher-worker-1`. Whereas, when resumed, it ran on thread `DefaultDispatcher-worker-2`. Coroutines run on configurable thread pools. As the log suggested, the default thread pool is called `Dispatchers.Default` (more on the dedicated following section).
+
+Last but not least, the log shows a clear example of structural concurrency. The execution printed the last log in the `main` method after the execution of both the coroutines. As we may have noticed, we didn't have any explicit synchronization mechanism to achieve this result in the `main` function. We didn't wait or delay the execution of the `main` function. As we said, this is due to structural concurrency. The `coroutineScope` function creates a scope that is used to create both the two coroutines. Since the two coroutines are children of the same scope, it will wait until the end of the execution of both of them before returning.
+
+We can also avoid using structural concurrency. In this case, we need to add some wait for the end of the execution of the coroutines. Instead of using the `coroutineScope` function, we can use the `GlobalScope` object. It's like **an empty coroutine scope that does not force any parent-child relationship**. So, we can rewrite the morning routine function as follows:
+
+```kotlin
+suspend fun noStructuralConcurrencyMorningRoutine() {
+ GlobalScope.launch {
+ bathTime()
+ }
+ GlobalScope.launch {
+ boilingWater()
+ }
+ Thread.sleep(1500L)
+}
+```
+
+The log of the above code is more or less the same as the previous one:
+
+```text
+14:06:57.670 [main] INFO CoroutinesPlayground - Starting the morning routine
+14:06:57.755 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Boiling water
+14:06:57.755 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Going to the bathroom
+14:06:58.264 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Exiting the bathroom
+14:06:58.763 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Water boiled
+14:06:59.257 [main] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+Since we do not have any structural concurrency mechanism using the `GlobalScope`, we added a `Thread.sleep(1500L)` at the end of the function to wait the end of the execution of the two coroutines. If we remove the `Thread.sleep` call, the log will be something similar to the following:
+
+```text
+21:47:09.418 [main] INFO CoroutinesPlayground - Starting the morning routine
+21:47:09.506 [main] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+As expected, the primary function returned before the end of the execution of the two coroutines. So, we can say that the `GlobalScope` is not a good choice for creating coroutines.
+
+If we look at the definition of the `launch` function, we can see that it returns a `Job` object. **This object is a handle to the coroutine**. We can use it to cancel the execution of the coroutine or to wait for its completion. Let's see how we can use it to wait for the coroutine's completion. Let's add a new suspending function to our wallet:
+
+```kotlin
+suspend fun preparingCoffee() {
+ logger.info("Preparing coffee")
+ delay(500L)
+ logger.info("Coffee prepared")
+}
+```
+
+In our morning routine, we only want to prepare coffee after a bath and boiling water. So, we need to wait for the completion of the two coroutines. We can do it by calling the `join` method on the resulting `Job` object:
+
+```kotlin
+suspend fun morningRoutineWithCoffee() {
+ coroutineScope {
+ val bathTimeJob: Job = launch {
+ bathTime()
+ }
+ val boilingWaterJob: Job = launch {
+ boilingWater()
+ }
+ bathTimeJob.join()
+ boilingWaterJob.join()
+ launch {
+ preparingCoffee()
+ }
+ }
+}
+```
+
+As expected, from the log, we can see that we prepared the coffee only after the end of the execution of the two coroutines:
+
+```text
+21:56:18.040 [main] INFO CoroutinesPlayground - Starting the morning routine
+21:56:18.128 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Going to the bathroom
+21:56:18.130 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Boiling water
+21:56:18.639 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Exiting the bathroom
+21:56:19.136 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Water boiled
+21:56:19.234 [DefaultDispatcher-worker-2 @coroutine#3] INFO CoroutinesPlayground - Preparing coffee
+21:56:19.739 [DefaultDispatcher-worker-2 @coroutine#3] INFO CoroutinesPlayground - Coffee prepared
+21:56:19.739 [DefaultDispatcher-worker-2 @coroutine#3] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+However, since we know all the secrets of structural concurrency now, we can rewrite the above code using the power of the `coroutineScope` function:
+
+```kotlin
+suspend fun structuralConcurrentMorningRoutineWithCoffee() {
+ coroutineScope {
+ coroutineScope {
+ launch {
+ bathTime()
+ }
+ launch {
+ boilingWater()
+ }
+ }
+ launch {
+ preparingCoffee()
+ }
+ }
+}
+```
+
+The output of the above code is the same as the previous one.
+
+### 5.2. The `async` Builder
+
+What if we want to return a value from the execution of a coroutine? For example, let's define two new suspending functions: The former produces the blend of the coffee we prepared. At the same time, the latter returns a toasted bread:
+
+```kotlin
+suspend fun preparingJavaCoffee(): String {
+ logger.info("Preparing coffee")
+ delay(500L)
+ logger.info("Coffee prepared")
+ return "Java coffee"
+}
+
+suspend fun toastingBread(): String {
+ logger.info("Toasting bread")
+ delay(1000L)
+ logger.info("Bread toasted")
+ return "Toasted bread"
+}
+```
+
+Fortunately, the library provides a way for a coroutine to return a value. **We can use the `async` builder to create a coroutine that returns a value**. In detail, it produces a value of type `Deferred`, which acts more or less like a java `Future`. On the object of type `Deferred`, we can call the `await` method to wait for the coroutine's completion and get the returned value. The library also defines the `async` builder as a `CoroutineScope` extension method:
+
+```kotlin
+public fun CoroutineScope.async(
+ context: CoroutineContext = EmptyCoroutineContext,
+ start: CoroutineStart = CoroutineStart.DEFAULT,
+ block: suspend CoroutineScope.() -> T
+): Deferred
+```
+
+Let's see how we can use it to return the blend of the coffee we prepared and the toasted bread:
+
+```kotlin
+suspend fun breakfastPreparation() {
+ coroutineScope {
+ val coffee: Deferred = async {
+ preparingJavaCoffee()
+ }
+ val toast: Deferred = async {
+ toastingBread()
+ }
+ logger.info("I'm eating ${coffee.await()} and ${toast.await()}")
+ }
+}
+```
+
+If we look at the log, we can see that the execution of the two coroutines is still concurrent. The last log awaits the completion of the two coroutines to print the final message:
+
+```text
+21:56:46.091 [main] INFO CoroutinesPlayground - Starting the morning routine
+21:56:46.253 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Preparing coffee
+21:56:46.258 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Toasting bread
+21:56:46.758 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Coffee prepared
+21:56:47.263 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Bread toasted
+21:56:47.263 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - I'm eating Java coffee and Toasted bread
+21:56:47.263 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+## 6. Cooperative Scheduling
+
+At this point, we should know something about the basics of coroutines. However, we still have to discuss one essential coroutines' aspect: cooperative scheduling.
+
+The coroutines scheduling model is very different from the one adopted by Java `Threads`, called preemptive scheduling. In preemptive scheduling, the operating system decides when to switch from one thread to another. **In cooperative scheduling, the coroutine itself decides when to yield the control to another coroutine**.
+
+In the case of Kotlin, a coroutine decides to yield the control reaching a suspending function. Only at that moment the thread executing it will be released and allowed to run another coroutine.
+
+If we noticed, in the logs we've seen so far, the execution control always changed when calling the `delay` suspending function. However, to understand it better, let's see another example. Let's define a new suspending function that simulates the execution of a very long-running task:
+
+```kotlin
+suspend fun workingHard() {
+ logger.info("Working")
+ while (true) {
+ // Do nothing
+ }
+ delay(100L)
+ logger.info("Work done")
+}
+```
+
+The infinite cycle will prevent the function from reaching the `delay` suspending function, so the coroutine will never yield control. Now, we define another suspending function to execute concurrently with the previous one:
+
+```kotlin
+suspend fun takeABreak() {
+ logger.info("Taking a break")
+ delay(1000L)
+ logger.info("Break done")
+}
+```
+
+Finally, let's glue everything together in a new suspending function running the two previous functions in two dedicated coroutines. To make sure we'll see the effect of the cooperative scheduling, we limit the thread pool executing the coroutines to a single thread:
+
+```kotlin
+@OptIn(ExperimentalCoroutinesApi::class)
+suspend fun workingHardRoutine() {
+ val dispatcher: CoroutineDispatcher = Dispatchers.Default.limitedParallelism(1)
+ coroutineScope {
+ launch(dispatcher) {
+ workingHard()
+ }
+ launch(dispatcher) {
+ takeABreak()
+ }
+ }
+}
+```
+
+The `CoroutineDispatcher` represents the thread pool used to execute the coroutines. The `limitedParallelism` function is an extension method of the `CoroutineDispatcher` interface that limits the number of threads in the thread pool to the given value. Since it's an experimental API, we need to annotate the function with the `@OptIn(ExperimentalCoroutinesApi::class)` annotation to avoid compiler warnings.
+
+We launched both the coroutines on the only available thread of the `dispatcher`, and the log shows us the effect of the cooperative scheduling:
+
+```text
+08:46:04.804 [main] INFO CoroutinesPlayground - Starting the morning routine
+08:46:04.884 [DefaultDispatcher-worker-2 @coroutine#1] INFO CoroutinesPlayground - Working
+-- Running forever --
+```
+
+Since the `workingHard` coroutine never reached a suspending function, it never yields the control back. Then, the `takeABreak` coroutine is never executed. On the contrary, if we define a suspending function that yields the control back to the dispatcher, the `takeABreak` coroutine will have the chance to be executed:
+
+```kotlin
+suspend fun workingConsciousness() {
+ logger.info("Working")
+ while (true) {
+ delay(100L)
+ }
+ logger.info("Work done")
+}
+
+@OptIn(ExperimentalCoroutinesApi::class)
+suspend fun workingConsciousnessRoutine() {
+ val dispatcher: CoroutineDispatcher = Dispatchers.Default.limitedParallelism(1)
+ coroutineScope {
+ launch(dispatcher) {
+ workingConsciousness()
+ }
+ launch(dispatcher) {
+ takeABreak()
+ }
+ }
+}
+```
+
+Now, the log shows that the `takeABreak` coroutine had the chance to execute, even if the `workingConsciousness` runs forever and we have a single thread:
+
+```text
+09:02:49.302 [main] INFO CoroutinesPlayground - Starting the morning routine
+09:02:49.376 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Working
+09:02:49.382 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Taking a break
+09:02:50.387 [DefaultDispatcher-worker-1 @coroutine#2] INFO CoroutinesPlayground - Break done
+-- Running forever --
+```
+
+We can obtain the same log also using the `workingHard` coroutine, adding a thread to the thread pool:
+
+```kotlin
+@OptIn(ExperimentalCoroutinesApi::class)
+suspend fun workingHardRoutine() {
+ val dispatcher: CoroutineDispatcher = Dispatchers.Default.limitedParallelism(2)
+ coroutineScope {
+ launch(dispatcher) {
+ workingHard()
+ }
+ launch(dispatcher) {
+ takeABreak()
+ }
+ }
+}
+```
+
+Since we have two threads and two coroutines, the concurrency degree is now two. As usual, the log confirms the theory: `coroutine#1` executes on `DefaultDispatcher-worker-1`, and `coroutine#2` executes on `DefaultDispatcher-worker-2`.
+
+```text
+13:40:59.864 [main] INFO CoroutinesPlayground - Starting the morning routine
+13:40:59.998 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Working
+13:41:00.003 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Taking a break
+13:41:01.010 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Break done
+-- Running forever --
+```
+
+**Cooperative scheduling forces us to be very careful when designing our coroutines**. Suppose a coroutine performs an operation blocking the underlying thread, such as a JDBC call. In that case, it blocks the thread from executing any other coroutine.
+
+For this reason, the library allows us to use different dispatchers for different operations. The main ones are:
+
+1. `Dispatchers.Default` is the default dispatcher used by the library. It uses a thread pool with a number of threads equal to the number of available processors. It's the right choice for CPU-intensive operations.
+2. `Dispatchers.IO` is the dispatcher used for I/O operations. It uses a thread pool with a number of threads equal to available processors or, at most 64. It's the right choice for I/O operations, such as network calls or file operations.
+3. Dispatcher created from a thread pool: It's possible to make our instance of `CoroutineDispatcher` using a thread pool. We can easily use the `asCoroutineDispatcher` extension function of the `Executor` interface. However, be aware that it's our responsibility to close the underlying thread pool when we don't need it anymore:
+
+```kotlin
+val dispatcher = Executors.newFixedThreadPool(10).asCoroutineDispatcher()
+```
+
+If we have both CPU-intensive and blocking parts, we must use both the `Dispatchers.Default` and the `Dispatchers.IO` and make sure to launch CPU-intensive coroutines on the default dispatchers and blocking code on the IO dispatcher.
+
+## 7. Cancellation
+
+When we reason about concurrent programming, cancellation is always a tricky topic. Killing a thread and abruptly stopping the execution of a task is not a good practice. **Before stopping a task, we must free the resources in use, avoid leaks, and leave the system in a consistent state**.
+
+As we can imagine, Kotlin allows us to cancel the execution of coroutines. **The library provides a mechanism to cancel a coroutine cooperatively to avoid problems**. The `Job` type provides a `cancel` function that cancels the execution of the coroutine. However, the cancellation is not immediate and happens only when the coroutine reaches a suspending point. The mechanism is very close to the one we saw for cooperative scheduling.
+
+Let's see an example. We want to model that we receive an important call during the working routine. We forgot the birthday of our best friend, and we want to go to buy a present before the mall closes:
+
+```kotlin
+suspend fun forgettingTheBirthDayRoutine() {
+ coroutineScope {
+ val workingJob = launch {
+ workingConsciousness()
+ }
+ launch {
+ delay(2000L)
+ workingJob.cancel()
+ workingJob.join()
+ logger.info("I forgot the birthday! Let's go to the mall!")
+ }
+ }
+}
+```
+
+A lot is going on in this snippet. First, we started the `workingConsciousness` coroutine and collected the corresponding `Job`. We used the `workingConsciousness` suspending function because it suspends inside the infinite loop, calling the `delay` function.
+
+Concurrently, we launch another coroutine, which cancels the `workingJob` after 2 seconds and waits for its completion. The `workingJob` is canceled, but the `workingConsciousness` coroutine is not stopped immediately. It continues to execute until it reaches the suspending point, and then it is canceled. Since we want to wait for the cancellation, we call the `join` function on the `workingJob`.
+
+The log confirms the theory. About 2 seconds from the start of the `coroutine#1`, the `coroutine#2` prints its log, and the `coroutine#1` is canceled:
+
+```text
+21:36:04.205 [main] INFO CoroutinesPlayground - Starting the morning routine
+21:36:04.278 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Working
+21:36:06.390 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - I forgot the birthday! Let's go to the mall!
+21:36:06.391 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+The `cancel` and then `join` pattern is so common that the Kotlin coroutines library provides us with a `cancelAndJoin` function that combines the two operations.
+
+As we said, cancellation is a cooperative affair in Kotlin. **If a coroutine never suspends, it cannot be canceled at all**. Let's change the above example using the `workingHard` suspending function instead. In this case, the `workingHard` function never suspends, so we expect the `workingJob` cannot be canceled:
+
+```kotlin
+suspend fun forgettingTheBirthDayRoutineWhileWorkingHard() {
+ coroutineScope {
+ val workingJob = launch {
+ workingHard()
+ }
+ launch {
+ delay(2000L)
+ workingJob.cancelAndJoin()
+ logger.info("I forgot the birthday! Let's go to the mall!")
+ }
+ }
+}
+```
+
+This time, our friend will not receive her present. The `workingJob` is canceled, but the `workingHard` function is not stopped since it never reaches a suspension point. Again, the log confirms the theory:
+
+```text
+08:56:10.784 [main] INFO CoroutinesPlayground - Starting the morning routine
+08:56:10.849 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Working
+-- Running forever --
+```
+
+Behind the scenes, the `cancel` function sets the `Job` in a state called "Cancelling". At first reached suspension point, the runtime throws a `CancellationException`, and the coroutine is finally canceled. This mechanism allows us to clean up the resources used by the coroutine safely. There are a lot of strategies we can implement to clean up the resources, but first, we need a resource to free during our examples. We can define the class `Desk` that represents a desk in our office:
+
+```kotlin
+class Desk : AutoCloseable {
+ init {
+ logger.info("Starting to work on the desk")
+ }
+
+ override fun close() {
+ logger.info("Cleaning the desk")
+ }
+}
+```
+
+The `Desk` class implements the `AutoCloseable` interface. So, it's an excellent candidate to free during a coroutine's cancellation. Since it implements `AutoCloseable`, we can use the `use` function to automatically close the resource when the block of code is completed:
+
+```kotlin
+suspend fun forgettingTheBirthDayRoutineAndCleaningTheDesk() {
+ val desk = Desk()
+ coroutineScope {
+ val workingJob = launch {
+ desk.use { _ ->
+ workingConsciousness()
+ }
+ }
+ launch {
+ delay(2000L)
+ workingJob.cancelAndJoin()
+ logger.info("I forgot the birthday! Let's go to the mall!")
+ }
+ }
+}
+```
+
+The `use` function works precisely as the _try-with-resources_ construct in Java.
+
+As expected, before we moved to the mall, we cleaned up the desk, and the log confirms it:
+
+```text
+21:38:30.117 [main] INFO CoroutinesPlayground - Starting the morning routine
+21:38:30.124 [main] INFO CoroutinesPlayground - Starting to work on the desk
+21:38:30.226 [DefaultDispatcher-worker-1 @coroutine#1] INFO CoroutinesPlayground - Working
+21:38:32.298 [DefaultDispatcher-worker-2 @coroutine#1] INFO CoroutinesPlayground - Cleaning the desk
+21:38:32.298 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - I forgot the birthday! Let's go to the mall!
+21:38:32.298 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+We can also use the `invokeOnCompletion` function on the canceling `Job` to clean up the desk after the `workingConsciousness` function is completed:
+
+```kotlin
+suspend fun forgettingTheBirthDayRoutineAndCleaningTheDeskOnCompletion() {
+ val desk = Desk()
+ coroutineScope {
+ val workingJob = launch {
+ workingConsciousness()
+ }
+ workingJob.invokeOnCompletion { exception: Throwable? ->
+ desk.close()
+ }
+ launch {
+ delay(2000L)
+ workingJob.cancelAndJoin()
+ logger.info("I forgot the birthday! Let's go to the mall!")
+ }
+ }
+}
+```
+
+As we can see, the `invokeOnCompletion` method takes a nullable exception as an input argument. If the `Job` is canceled, the exception is a `CancellationException`.
+
+**Another feature of cancellation is it propagates to children coroutines**. When we cancel a coroutine, we implicitly cancel all of its children. Let's see an example. During the day, it's essential to stay hydrated. We can use the `drinkWater` suspending function to drink water:
+
+```kotlin
+suspend fun drinkWater() {
+ while (true) {
+ logger.info("Drinking water")
+ delay(1000L)
+ logger.info("Water drunk")
+ }
+}
+```
+
+Then, we can create a coroutine that spawns two new coroutines for working and drinking water. Finally, we can cancel the parent coroutine, and we expect that the two children are canceled as well:
+
+```kotlin
+suspend fun forgettingTheBirthDayWhileWorkingAndDrinkingWaterRoutine() {
+ coroutineScope {
+ val workingJob = launch {
+ launch {
+ workingConsciousness()
+ }
+ launch {
+ drinkWater()
+ }
+ }
+ launch {
+ delay(2000L)
+ workingJob.cancelAndJoin()
+ logger.info("I forgot the birthday! Let's go to the mall!")
+ }
+ }
+}
+```
+
+As expected, when we cancel the `workingJob`, we also cancel and stop its children's coroutines. Here is the log that describes the situation:
+
+```text
+13:18:49.143 [main] INFO CoroutinesPlayground - Starting the morning routine
+13:18:49.275 [DefaultDispatcher-worker-2 @coroutine#2] INFO CoroutinesPlayground - Working
+13:18:49.285 [DefaultDispatcher-worker-3 @coroutine#3] INFO CoroutinesPlayground - Drinking water
+13:18:50.285 [DefaultDispatcher-worker-3 @coroutine#3] INFO CoroutinesPlayground - Water drunk
+13:18:50.286 [DefaultDispatcher-worker-3 @coroutine#3] INFO CoroutinesPlayground - Drinking water
+13:18:51.288 [DefaultDispatcher-worker-2 @coroutine#3] INFO CoroutinesPlayground - Water drunk
+13:18:51.288 [DefaultDispatcher-worker-2 @coroutine#3] INFO CoroutinesPlayground - Drinking water
+13:18:51.357 [DefaultDispatcher-worker-2 @coroutine#4] INFO CoroutinesPlayground - I forgot the birthday! Let's go to the mall!
+13:18:51.357 [DefaultDispatcher-worker-2 @coroutine#4] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+And that's all for coroutines cancellation!
+
+## 8. The Coroutine Context
+
+In the section concerning continuation and the section concerning builders, we briefly introduced the concept of coroutine context. Also, the `CoroutineScope` retains a reference to a coroutine context. As you can imagine, **it is a way to store information passed from parents to children to develop structural concurrency internally**.
+
+The type representing the coroutine context is called `CoroutineContext`, and it is part of the Kotlin core library. It's a funny type since it represents a collection of elements, but also, every element is a collection:
+
+```kotlin
+public interface CoroutineContext
+// But also
+public interface Element : CoroutineContext
+```
+
+The implementation of the `CoroutineContext` is placed in the Kotlin coroutines library, together with the `Continuation` type. Among the actual implementations, we have the `CoroutineName`, which represents the name of a coroutine:
+
+```kotlin
+val name: CoroutineContext = CoroutineName("Morning Routine")
+```
+
+In addition, the `CoroutineDispatcher` and the `Job` type implement the `CoroutineContext` interface. The identifier we saw in the above logs is the `CoroutineId`. This context is automatically added by the runtime to every coroutine when we enable the debug mode.
+
+Since the `CoroutineContext` behaves like a collection, the library also defines the `+` operator to add elements to the context. So, creating a new context with many elements is as simple as:
+
+```kotlin
+val context: CoroutineContext = CoroutineName("Morning Routine") + Dispatchers.Default + Job()
+```
+
+Removing elements from the context is also possible using the `minusKey` function:
+
+```kotlin
+val newContext: CoroutineContext = context.minusKey(CoroutineName)
+```
+
+As we should remember, we can pass the context to a builder to change the behavior of the created coroutine. For example, suppose we want to create a coroutine with a specific name that uses the `Dispatchers.Default`. In that case, we can do it as follows:
+
+```kotlin
+suspend fun asynchronousGreeting() {
+ coroutineScope {
+ launch(CoroutineName("Greeting Coroutine") + Dispatchers.Default) {
+ logger.info("Hello Everyone!")
+ }
+ }
+}
+```
+
+Let's run it inside the `main` function. We can see in the log that the coroutine is created with the specified name, and it's executed in the `Default` dispatcher:
+
+```text
+11:56:46.747 [DefaultDispatcher-worker-1 @Greeting Coroutine#1] INFO CoroutinesPlayground - Hello Everyone!
+```
+
+A coroutine context also behaves as a map since we can search and access the elements it contains using the name of the type corresponding to the element we want to retrieve:
+
+```kotlin
+logger.info("Coroutine name: {}", context[CoroutineName]?.name)
+```
+
+The above code prints the coroutine name stored in the context, if any. The `CoroutineName` used inside the square brackets is neither a type nor a class. Indeed, it references the companion object called the `Key` of the class—just some Kotlin syntactic sugar.
+
+The library also defines the empty coroutine context, `EmptyCoroutineContext`, which we can use as a "zero" element to create a new custom context.
+
+So, context is a way to pass information among coroutines. **Any parent coroutine gives its context to its children coroutines**. Children coroutines copy values from the parent to a new instance of the context that they can override. Let's see an example of inheritance without override:
+
+```kotlin
+suspend fun coroutineCtxInheritance() {
+ coroutineScope {
+ launch(CoroutineName("Greeting Coroutine")) {
+ logger.info("Hello everyone from the outer coroutine!")
+ launch {
+ logger.info("Hello everyone from the inner coroutine!")
+ }
+ delay(200L)
+ logger.info("Hello again from the outer coroutine!")
+ }
+ }
+}
+```
+
+The log of the above code is the following, and it highlights that both coroutines share the same name:
+
+```text
+12:19:12.962 [DefaultDispatcher-worker-1 @Greeting Coroutine#1] INFO CoroutinesPlayground - Hello everyone from the outer coroutine!
+12:19:12.963 [DefaultDispatcher-worker-2 @Greeting Coroutine#2] INFO CoroutinesPlayground - Hello everyone from the inner coroutine!
+12:19:12.963 [DefaultDispatcher-worker-1 @Greeting Coroutine#1] INFO CoroutinesPlayground - Hello again from the outer coroutine!
+```
+
+As we said, if we want, we can override the values inside the context from the child coroutine:
+
+```kotlin
+suspend fun coroutineCtxOverride() {
+ coroutineScope {
+ launch(CoroutineName("Greeting Coroutine")) {
+ logger.info("Hello everyone from the outer coroutine!")
+ launch(CoroutineName("Greeting Inner Coroutine")) {
+ logger.info("Hello everyone from the inner coroutine!")
+ }
+ delay(200L)
+ logger.info("Hello again from the outer coroutine!")
+ }
+ }
+}
+```
+
+The log of the above code shows the override of the parent coroutine. However, the original value is still the original in the parent context:
+
+```text
+12:22:33.869 [DefaultDispatcher-worker-1 @Greeting Coroutine#1] INFO CoroutinesPlayground - Hello everyone from the outer coroutine!
+12:22:33.870 [DefaultDispatcher-worker-2 @Greeting Inner Coroutine#2] INFO CoroutinesPlayground - Hello everyone from the inner coroutine!
+12:22:34.077 [DefaultDispatcher-worker-1 @Greeting Coroutine#1] INFO CoroutinesPlayground - Hello again from the outer coroutine!
+12:22:34.078 [DefaultDispatcher-worker-1 @Greeting Coroutine#1] INFO CoroutinesPlayground - Ending the morning routine
+```
+
+The only exception to the context inheritance rule is the `Job` context instance. Every new coroutine creates its own `Job` instance, which is not inherited from the parent. Whereas, the other context elements, such as the `CoroutineName` or the dispatcher, are inherited from the parent.
+
+## 9. Conclusions
+
+Our journey through the basics of the Kotlin coroutines library is over. We saw why coroutines matter and made a simplified explanation of how they're implemented under the hood. Then, we showed how to create coroutines, also introducing the structural concurrency topic. We saw how cooperative scheduling and cancellation work with many examples. Finally, we introduced the main features of the coroutines' context. There is a lot more to say about coroutines, but we hope this article can be a good starting point for those who want to learn more about them.
+
+If you found coroutines too difficult, you can quickly get the Kotlin basics you need by following the complete [Kotlin Essentials course](https://rockthejvm.com/p/kotlin-essentials) on Rock the JVM.
+
+## 10. Appendix A
+
+As promised, here is the `pom.xml` file that we used to run the code in this article:
+
+```xml
+
+
+ 4.0.0
+
+ in.rcard
+ kactor-coroutines-playground
+ 0.0.1-SNAPSHOT
+
+
+ 1.7.20
+ 1.6.4
+ 2.0.5
+ 1.4.5
+
+
+
+
+ org.jetbrains.kotlin
+ kotlin-stdlib
+ ${kotlin.version}
+
+
+ org.jetbrains.kotlinx
+ kotlinx-coroutines-core
+ ${kotlinx-coroutines.version}
+
+
+
+ org.slf4j
+ slf4j-api
+ ${slf4j-api.version}
+
+
+ ch.qos.logback
+ logback-classic
+ ${logback-classic.version}
+
+
+
+
+ ${project.basedir}/src/main/kotlin
+ ${project.basedir}/src/test/kotlin
+
+
+
+ org.jetbrains.kotlin
+ kotlin-maven-plugin
+ ${kotlin.version}
+
+
+
+ compile
+
+ compile
+
+
+
+
+ test-compile
+
+ test-compile
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+ 7
+ 7
+
+
+
+
+
+```
+
+Enjoy!
diff --git a/_posts/2023-01-09-scala-cli-and-scala-native.md b/_posts/2023-01-09-scala-cli-and-scala-native.md
new file mode 100644
index 000000000000..eef3cb805afb
--- /dev/null
+++ b/_posts/2023-01-09-scala-cli-and-scala-native.md
@@ -0,0 +1,708 @@
+---
+title: "Scala CLI Tutorial: Creating a CLI Sudoku Solver"
+date: 2023-01-09
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala 3, cats, scala-native, scala-cli]
+excerpt: "Scala CLI is a great tool for prototyping and building Scala applications. We'll use `scala-cli`, `Scala Native` and `decline` to build a brute-force sudoku solver."
+---
+
+_This article is brought to you by [Antonio Gelameris](https://github.com/TonioGela). Antonio is an alumnus of Rock the JVM, now a senior Scala developer with his own contributions to Scala libraries and junior devs under his mentorship.
+Which brings us to this article: Antonio originally started from my [Sudoku backtracking](/sudoku-backtracking) article and built a Scala CLI tutorial for the juniors he's mentoring. Now, he's sharing his process with us._
+
+_Enter Antonio:_
+
+## 1. Introduction
+
+[Sudoku](https://en.wikipedia.org/wiki/Sudoku) is a notorious combinatorial puzzle solvable with optimised and efficient algorithms. Today we won't focus on any of those techniques, but we'll leverage the computing power of our machines to brute-force the solution in a functional immutable fashion.
+
+The Scala ecosystem has many fantastic tools and libraries to help us synthesise the solution and package our solver in an ultra-fast native executable with instant startup time using our favourite language and its expressive power. To implement our solution, I chose [scala-cli](https://scala-cli.virtuslab.org/) to structure the project and to compile it with [Scala Native](https://scala-native.org/en/stable/), [decline](https://ben.kirw.in/decline/) to parse command line arguments and [cats](https://typelevel.org/cats/) for its purely functional approach.
+
+## 2. Scala-CLI: your best command line buddy
+
+[Scala CLI](https://scala-cli.virtuslab.org/) is a recent command line tool by [VirtusLab](https://virtuslab.org/) that lets you interact with Scala in multiple ways. One of its most valuable features is the support to create single-file scripts that can use any Scala dependency and be packaged in various formats to run everywhere.
+
+Once [installed](https://scala-cli.virtuslab.org/install), let's write a simple hello world application in a `.scala` file :
+
+```scala
+/* Hello.scala */
+object Hello {
+ def main(args: Array[String]): Unit = println("Hello from scala-cli")
+}
+```
+
+and run it using `scala-cli run Hello.scala`
+
+```shell
+$ scala-cli run Hello.scala
+# Compiling project (Scala 3.2.0, JVM)
+# Compiled project (Scala 3.2.0, JVM)
+Hello from scala-cli
+```
+
+Scala CLI, by default, downloads the latest scala version and uses the available JVM installed on your system unless you specify otherwise.
+
+```shell
+$ scala-cli run Hello.scala --jvm "temurin:11" --scala "2.13.10"
+# Downloading JVM temurin:11
+# Compiling project (Scala 2.13.10, JVM)
+# Compiled project (Scala 2.13.10, JVM)
+Hello from scala-cli
+```
+The best way to customise its default behaviour is through Scala CLI's [using Directives](https://scala-cli.virtuslab.org/docs/guides/using-directives).
+
+### 2.1 Directives
+Let's say that for the purposes of our script, a library like [PPrint](https://github.com/com-lihaoyi/PPrint) might be convenient. With directives, it's possible to declare it as our script's dependency and to specify both the JVM and Scala versions we intend to run our script with:
+
+```scala
+/* Maps.scala */
+//> using scala "2.13.10"
+//> using jvm "temurin:11"
+//> using lib "com.lihaoyi::pprint::0.6.6"
+
+object Maps {
+ def main(args: Array[String]): Unit =
+ println("Maps in Scala have the shape " + pprint.tprint[Map[_,_]])
+}
+```
+
+Now it's possible to execute the script with no additional command line flags
+
+```shell
+$ scala-cli run Hello.scala
+# Compiling project (Scala 2.13.10, JVM)
+# Compiled project (Scala 2.13.10, JVM)
+Maps in Scala have the shape Map[_, _]
+```
+
+Through directives you can, for example:
+- add java options or compiler flags
+- declare tests
+- change the compilation target
+- package the application as a fat jar or as a script that downloads all the required dependencies
+
+and much more. For a complete reference, see [Directives](https://scala-cli.virtuslab.org/docs/reference/scala-command/directives).
+
+### 2.2. Updating dependencies
+
+As some of you may have noticed, the `pprint` library version in the example is not the newest one: at the time of writing, the most recent version is 0.8.0. Luckily we're not forced to _check it manually on Github or Maven Central_ since scala-cli exposes the `dependency-update` command that will fetch the last version of each dependency and print a command to update them all.
+
+```shell
+$ scala-cli dependency-update Maps.scala
+Updates
+ * com.lihaoyi::pprint::0.6.6 -> 0.8.0
+To update all dependencies run:
+ scala-cli dependency-update --all
+
+$ scala-cli dependency-update --all Maps.scala
+Updated dependency to: com.lihaoyi::pprint::0.8.0
+
+$ head -3 Maps.scala
+//> using scala "2.13.10"
+//> using jvm "temurin:11"
+//> using lib "com.lihaoyi::pprint::0.8.0"
+```
+
+### 2.3. IDE support
+
+Writing Scala code without the help of a fully-fledged IDE is okay if you're writing a "Hello world" application or similar, but for a _"complete programming experience"_ using one of the IDE alternatives — at the moment either IntelliJ or a Metals-compatible one — is recommended. Scala CLI can help you set up your IDE of choice by generating the necessary files to provide full-blown IDE support.
+
+The [setup-ide](https://scala-cli.virtuslab.org/docs/commands/setup-ide) command is run before every `run`, `compile` or `test` but it can be invoked manually like:
+
+```shell
+$ scala-cli setup-ide Maps.scala
+```
+
+resulting in the generation of 2 files that both Metals and IntelliJ use to provide all their functionalities.
+
+```shell
+.
+├── .bsp
+│ └── scala-cli.json
+├── .scala-build
+│ └── ide-inputs.json
+└── Maps.scala
+```
+
+Opening the _enclosing folder_ in your Metals-enabled editor or importing it in IntelliJ will provide you with the Scala IDE experience you're used to.
+
+### 2.4. Formatting
+
+Our developer experience can't be complete without a properly configured formatter. Luckily scala-cli can run [scalafmt](https://scalameta.org/scalafmt/) with `scala-cli fmt Maps.scala`. A `.scalafmt.conf` file in the project's root folder will let you customize the default formatting behaviour (add `--save-scalafmt-conf` to save locally the default configuration if needed).
+
+Now that we have a working IDE, we can begin modelling the problem and its solution.
+
+## 3. Modeling a Sudoku Board
+
+Since sudoku consists of 9 lines of 9 digits from 1 to 9, one of the ways to encode and store the information in a case class is wrapping a `Vector[Int]`. So in a newly created `Sudoku.scala` file, we'll define
+
+```scala
+/* Sudoku.scala */
+//> using scala "3.2.1"
+
+final case class Sudoku private (data: Vector[Int])
+```
+
+We made the constructor private to avoid `Sudoku` getting instantiated outside its companion object, where we will soon create a "factory" method named `from`.
+
+Since we plan to read sudoku boards from the command line, it's reasonable to imagine a factory method that accepts a `String` and returns a `Sudoku` or a data structure that may contain either a `Sudoku` or a way to signal an error (like an error `String` to log in case of validation errors).
+
+```scala
+/* Sudoku.scala */
+//> using scala "3.2.1"
+
+final case class Sudoku private (data: Vector[Int])
+
+object Sudoku {
+
+ def from(s: String): Either[String, Sudoku] = ???
+}
+```
+
+To implement the method, we'll leverage some utility functions that [Cats](https://typelevel.org/cats/) provide.
+
+```scala
+/* Sudoku.scala */
+//> using scala "3.2.1"
+//> using lib "org.typelevel::cats-core::2.9.0"
+
+import cats.syntax.all.*
+
+final case class Sudoku private (data: Vector[Int])
+
+object Sudoku {
+
+ def from(s: String): Either[String, Sudoku] =
+ s.replace('.', '0')
+ .asRight[String]
+ .ensure("The sudoku string doesn't contain only digits")(
+ _.forall(_.isDigit)
+ )
+ .map(_.toVector.map(_.asDigit))
+ .ensure("The sudoku string is not exactly 81 characters long")(
+ _.length === 81
+ )
+ .map(Sudoku.apply)
+}
+```
+
+Let's examine the `from` function line by line:
+- `s.replace('.', '0')` replaces the `.`s with `0`s to signal the lack of a digit using a value that belongs to type `Int`. Replacing `.` is necessary since we'll use [this generator](https://qqwing.com/generate.html) with "Output format: One line", getting an input value like `8...1...2.7...931.....485...2....8.91..2....3.........7...9...1.5...1.....3.7.29.` that represents the 81 digits of the board.
+- `.asRight[String]` is the first cats utility that we'll use. Defined as
+
+ ```scala
+ def asRight[B]: Either[B, A] = Right(a)
+ ```
+
+ it is an extension method over `a: A`. It wraps the value in a `Right` but requires a type argument `B` to widen the result declaration to `Either[B,A]`. This way, the result will not have type `Right[String]` but `Either[String, String]`, letting us use other utility functions defined over `Either[_,_]`.
+- `.ensure("The sudoku string doesn't contain only digits")(_.forall(_.isDigit))` uses the extension method `ensure`, a guard function that filters either in the case is a `Right` and returns the content of the first parenthesis in case of errors. Its definition (where `eab` is the extended value) is
+ ```scala
+ def ensure(onFailure: => A)(condition: B => Boolean): Either[A, B] = eab match {
+ case Left(_) => eab
+ case Right(b) => if (condition(b)) eab else Left(onFailure)
+ }
+ ```
+ In this particular case, we use to check that all the characters in the string (`forall`) are digits (`isDigit`) otherwise, we return a `Left("The sudoku string doesn't contain only digits")` to signal the error, short-circuiting all the following validations.
+- `.map(_.toVector.map(_.asDigit))` maps over the `Either[String,String]` to transform its content (when it's a `Right`) and then we map every `Char` into an `Int` `map`ping over the vector. (Note: we use `asDigit` and not `toDigit` as we want to interpret the literal value of the `Char` as a digit and not its internal representation)
+- Using the same `ensure` function we check that the string has the correct length
+- Finally, we map the `Either[String, Vector[Int]]` into an `Either[String, Sudoku]` calling `Sudoku`'s constructor, that here in the companion object is accessible.
+
+The main strength of the `from` function is that it won't let us create a `Sudoku` if the input doesn't comply with a set of minimum requirements needed to fully and correctly describe a `Sudoku`. This approach, sometimes called ["Parse, don't validate"](https://lexi-lambda.github.io/blog/2019/11/05/parse-don-t-validate/), might not seem like a big deal. Still, it enables us to write functions and extension methods that use `Sudoku` as parameters and are not required to perform any validation. `Sudoku`s are now impossible to create without using a valid input: we made invalid `Sudoku`s impossible to represent.
+
+## 4. Adding utility methods
+
+Our `Sudoku` case class is pretty much useless without any function using it, so let's write a few methods that might help us solve the problem. Since each number in each cell is _row_, _column_ and _cell_ constrained, it makes sense to code a way to extract those pieces of information from the case class.
+
+```scala
+/* Sudoku.scala */
+//> using scala "3.2.1"
+//> using lib "org.typelevel::cats-core::2.9.0"
+
+import cats.syntax.all.*
+
+final case class Sudoku private (data: Vector[Int]) {
+
+ def get(x: Int)(y: Int): Int = data(y * 9 + x)
+
+ def getRow(y: Int): Vector[Int] = data.slice(y * 9, (y + 1) * 9)
+
+ def getColumn(x: Int): Vector[Int] = (0 until 9).toVector.map(get(x))
+
+ def getCellOf(x: Int)(y: Int): Vector[Int] = {
+ def span(n: Int): Vector[Int] = {
+ val x: Int = (3 * (n / 3))
+ Vector(x, x + 1, x + 2)
+ }
+
+ for {
+ b <- span(y)
+ a <- span(x)
+ } yield get(a)(b)
+ }
+}
+
+object Sudoku {
+
+ def from(s: String): Either[String, Sudoku] = /* ... */
+}
+```
+
+> Note: We added these methods to the case class itself, but another option we could have chosen is to add this logic in an [extension](https://docs.scala-lang.org/scala3/book/ca-extension-methods.html). Creating an extension over the `Sudoku` datatype will let us call the methods defined in it as if they were methods of the `Sudoku` class.
+> ```scala
+> object Sudoku {
+>
+> extension (s: Sudoku) {
+> def get(x: Int)(y: Int): Int = /* ... */
+> def getRow(y: Int): Vector[Int] = /* ... */
+> /* ... */
+> }
+> }
+>
+> sudoku.get(0)(0)
+> ```
+> Extending may be preferable since it keeps data separated from the logic that manipulates them (enforcing some [separation of concerns](https://en.wikipedia.org/wiki/Separation_of_concerns)), and since it's possible over data types not part of your codebase, like standard library types or data types coming from a library. This approach shines when the extension depends on a [typeclass](https://docs.scala-lang.org/scala3/book/ca-type-classes.html), since extending the type class for a new type `T` (i.e. adding a "*case*") you get a custom syntax over `T` for free.
+>
+> On the other hand, defining new methods in the class (or in a trait) is easier if you intend to add new operations to that specific type (or trait). Pros and cons of the _type class_ vs _inheritance_ approach to the [Wadler's expression problem](https://en.wikipedia.org/wiki/Expression_problem) will be discussed in a future article.
+
+## 5. Testing
+
+Now that we have some APIs over `Sudoku`, it makes sense to test them out before trying to solve the problem further. [Scala-cli supports testing](https://scala-cli.virtuslab.org/docs/commands/test) out of the box and detects test files in several ways. The easiest one to leverage is using the `.test.scala` extension, ideal when you have a single source file like `foo.scala` and its testing companion `foo.test.scala`.
+
+A more structured way to set up a project is to separate the source files from the test ones, maybe in different folder trees, using a slightly more complex project structure that [scala-cli supports](https://scala-cli.virtuslab.org/docs/reference/root-dir).
+
+```shell
+.
+├── src
+│ └── Sudoku.scala
+├── test
+│ └── SudokuSpec.scala
+└── project.scala
+```
+
+This structure auto-detects *test classes* using the files' relative path: if the file's path contains the string `"test"`, it will be treated as a test class. To test the application, we will declare munit in the `project.scala` file that now contains all the directives previously in `Sudoku.scala`.
+
+```scala
+/* project.scala */
+//> using scala "3.2.1"
+//> using lib "org.typelevel::cats-core::2.9.0"
+//> using lib "com.monovore::decline::2.4.1"
+//> using lib "org.scalameta::munit::0.7.29"
+```
+
+```scala
+/* src/Sudoku.scala */
+import cats.syntax.all.*
+
+final case class Sudoku private (data: Vector[Int]) { /* ... */ }
+
+object Sudoku { /* ... */ }
+```
+
+```scala
+/* test/SudokuSpec.scala */
+import munit.*
+import cats.syntax.all.*
+
+class SudokuSpec extends FunSuite {
+
+ // format: off
+ val maybeSudoku = Sudoku.from(
+ "435269781" +
+ "682571493" +
+ "197834562" +
+ "826195347" +
+ "374682915" +
+ "951743628" +
+ "519326874" +
+ "248957136" +
+ "763418259"
+ )
+ // format: on
+
+ val sudokuF: FunFixture[Sudoku] = FunFixture(_ => maybeSudoku.fold(failSuite(_), identity), _ => ())
+
+ /* Tests here! */
+
+}
+```
+
+To easily have a `Sudoku` instance available for easy unit testing, we used `FunFixture`. `FunFixture` is one of the [available fixtures](https://scalameta.org/munit/docs/fixtures.html) that munit provides to acquire and release resources and to share them between single tests or whole suites. We coded `sudokuF` to fail the entire suite if the `Sudoku` is trying to instantiate is invalid and to give it to the fixture user otherwise.
+
+Now we can define tests using the munit's simple syntax:
+
+```scala
+sudokuF.test("Sudoku.get(x,y) should extract the number at (x,y) (0 based, from top left)") { sudoku =>
+ assertEquals(sudoku.get(0)(0), 4)
+ assertEquals(sudoku.get(1)(0), 3)
+ assertEquals(sudoku.get(2)(7), 8)
+}
+
+sudokuF.test("Sudoku.getRow(n) should extract nth row from top") { sudoku =>
+ assertEquals(sudoku.getRow(0), Vector(4, 3, 5, 2, 6, 9, 7, 8, 1))
+ assertEquals(sudoku.getRow(6), Vector(5, 1, 9, 3, 2, 6, 8, 7, 4))
+}
+
+sudokuF.test("Sudoku.getColumn(n) should extract nth column from left") { sudoku =>
+ assertEquals(sudoku.getColumn(0), Vector(4, 6, 1, 8, 3, 9, 5, 2, 7))
+ assertEquals(sudoku.getColumn(6), Vector(7, 4, 5, 3, 9, 6, 8, 1, 2))
+}
+
+sudokuF.test("Sudoku.getCellOf(n) should extract the correct cell") { sudoku =>
+ assert(sudoku.getCellOf(1)(1).forall((1 to 9).contains))
+ assert(sudoku.getCellOf(7)(3).forall((1 to 9).contains))
+}
+```
+
+and test our implementation using the `test` command of scala-cli:
+
+```shell
+$ scala-cli test .
+SudokuSpec:
+ + Sudoku.get(x,y) should extract the number at (x,y) (0 based, from top left) 0.037s
+ + Sudoku.getRow(n) should extract nth row from top 0.002s
+ + Sudoku.getColumn(n) should extract nth column from left 0.001s
+ + Sudoku.getCellOf(n) should extract the correct cell 0.003s
+```
+
+## 6. Recursive immutable solution
+
+To solve the sudoku, we will use a recursive brute-forcing algorithm:
+
+1. Given a sudoku board, we will search for a zero
+2. For each zero, we will find all the numbers that fit in that position according to the constraints
+3. For each of those numbers, we will generate a new sudoku replacing the zero with it
+4. We will apply the three previous steps to every sudoku we have created so far until there are no more zeros
+5. We will end up with a list of solved sudoku boards that we will return to the user
+
+We will need to implement a couple of methods over `Sudoku` to implement this solution:
+
+```scala
+final case class Sudoku private (data: Vector[Int]) {
+
+ /* ... */
+
+ // None will signal the lack of zeros, so a complete sudoku.
+ // Since -1 means that indexWhere hasn't found zeros we remap
+ // it to None using filterNot.
+ def getZero: Option[(Int, Int)] = Option(data.indexWhere(_ === 0))
+ .filterNot(_ === -1)
+ .map(i => (i % 9, i / 9))
+
+ // This is the method that checks if the cell, row and
+ // column constraints are satisfied for a certain value
+ def fitsInPlace(x: Int, y: Int)(value: Int): Boolean =
+ !(getCellOf(x)(y).contains(value) || getRow(y).contains(value) || getColumn(x).contains(value))
+
+ def set(x: Int, y: Int)(value: Int): Sudoku = Sudoku(
+ data.updated(y * 9 + x, value)
+ )
+}
+```
+
+Let's try to implement the algorithm using the newly created methods. The function should accept a `Sudoku` and return all the possible solved ones, so the method signature is easy to write:
+
+```scala
+def solve(s: Sudoku): List[Sudoku] = ???
+```
+
+The first step is searching for zero, and we wrote a method for that:
+
+```scala
+def solve(s: Sudoku): List[Sudoku] = s.getZero match {
+ case None => ???
+ case Some((x,y)) => ???
+}
+```
+
+Since `getZero` returns a `None` in the case there are no more zeros in the sudoku, it means that `s` is solved, so we can return it to the caller, wrapping it in a `List` to comply with the function signature:
+
+```scala
+def solve(s: Sudoku): List[Sudoku] = s.getZero match {
+ case None => s :: Nil
+ case Some((x,y)) => ???
+}
+```
+
+In case `getZero` returns the coordinates of a zero, we have to calculate all the possible numbers that fit in that cell according to the constraints and return the list of the corresponding sudoku boards (with that zero replaced by a possible number). Since there are multiple ways to implement this logic, it makes sense to wrap it in a standalone function `calcStep`.
+
+Bear in mind that since this function returns a list of sudoku boards that satisfy some constraints, it may return an empty list. So this function is in charge of skimming the unsolvable boards from the list.
+
+```scala
+def solve(s: Sudoku): List[Sudoku] = s.getZero match {
+ case None => s :: Nil
+ case Some((x,y)) => calcStep(x, y)(s)
+}
+
+def calcStep(x: Int, y: Int)(s: Sudoku): List[Sudoku] = 1
+ .to(9)
+ .filter(s.fitsInPlace(x, y))
+ .map(s.set(x, y))
+ .toList
+```
+
+The function consists of 2 steps: filtering out from the 1 to 9 range the numbers that don't satisfy the constraints and getting a new `Sudoku` for each one that does.
+
+Now that the solving step has been implemented, it's time to add some recursion to find the solutions. Since the sudoku boards that `calcStep` returns might still have zeros, it makes sense to re-submit them to the `solve` function. Since we have a `List[Sudoku]` and `solve` returns a `List[Sudoku]` as well, the easiest way to chain the `solve` function to itself is using `flatMap`:
+
+```scala
+def solve(s: Sudoku): List[Sudoku] = s.getZero match {
+ case None => s :: Nil
+ case Some((x,y)) => calcStep(x, y)(s).flatMap(solve)
+}
+
+def calcStep(x: Int, y: Int)(s: Sudoku): List[Sudoku] = 1
+ .to(9)
+ .filter(s.fitsInPlace(x, y))
+ .map(s.set(x, y))
+ .toList
+```
+
+A fancier way to write this solution, leveraging some `cats` aliases and using `fold`, is the following:
+
+```scala
+def solve(s: Sudoku): List[Sudoku] = s.getZero.fold(s :: Nil) {
+ case (x, y) => calcStep(x, y)(s) >>= solve
+}
+```
+
+## 7. Creating the command line application
+
+Now that we've built the core of the logic, it's time to wire it to create a command line application. The chosen library for command line argument parsing is [decline](https://ben.kirw.in/decline/). We will place the argument parsing logic and the application entry point in their own `Main.scala` file.
+
+```shell
+.
+├── src
+│ ├── Main.scala
+│ └── Sudoku.scala
+├── test
+│ └── SudokuSpec.scala
+└── project.scala
+```
+
+The decline API to define a command line argument parser is called `Opts`. `Opts` features a few [basic options](https://ben.kirw.in/decline/usage.html#basic-options) to combine to determine the command line API of your application:
+
+- `Opts.argument[T]` to define a mandatory argument that MUST be passed to you application
+- `Opts.option[T]` to modify the program's behaviour and that require a value (like `-n 10`)
+- `Opts.flag` that are identical to `option` but don't require a value (like `-l` or `--verbose`)
+- `Opts.env[T]` to read environment variables
+
+Each of these options has an "s-terminating" alternative (like `Opts.arguments[T]`) that will parse multiple instances of the defined option. Decline features [commands and subcommands](https://ben.kirw.in/decline/usage.html#commands-and-subcommands), but they're out of scope for the sake of this post.
+
+Since our application's solving logic must receive a `Sudoku` we will write a `Opts[Sudoku]` definition:
+
+```scala
+/* src/Main.scala */
+import com.monovore.decline.*
+import cats.syntax.all.*
+import Sudoku.*
+
+val sudokuArgument: Opts[Sudoku] =
+ Opts.argument[String]("sudoku").mapValidated(Sudoku.from(_).toValidatedNel)
+```
+
+The definition starts from a string argument that gets parsed to create a `Sudoku`. Decline offers the `mapValidated` method, that accepts a `String => ValidatedNel[String, Sudoku]` function that should convert the provided string to a Sudoku. The returned data type is a [Validated](https://typelevel.org/cats/datatypes/validated), an `Either`-like structure offered by cats that doesn't form a monad and that is [particularly suited for error accumulation](https://typelevel.org/cats/datatypes/validated#parallel-validation). Luckily we can convert from `Either[A,B]` to `Validated[NonEmptyList[A],B]` using an extension method.
+
+To use the newly defined `sudokuArgument` we must extend [CommandApp](https://ben.kirw.in/decline/usage.html#using-commandapp), to wire up our app's `main` method:
+
+```scala
+/* src/Main.scala */
+import com.monovore.decline.*
+import cats.syntax.all.*
+import Sudoku.*
+
+val sudokuArgument: Opts[Sudoku] =
+ Opts.argument[String]("sudoku").mapValidated(Sudoku.from(_).toValidatedNel)
+
+object Main extends CommandApp(
+ name = "sudokuSolver",
+ header = "Solves sudokus passed as 81 chars string with . or 0 in place of empty cells",
+ main = sudokuArgument.map(sudoku => println(sudoku.asPrettyString))
+)
+```
+
+having this definition of `asPrettyString`:
+
+```scala
+ def asPrettyString: String = {
+ def showRow(a: Vector[Int]): String =
+ a.grouped(3).map(_.mkString).mkString("│")
+
+ (0 until 9)
+ .map(getRow)
+ .map(showRow)
+ .grouped(3)
+ .map(_.mkString("\n"))
+ .mkString("\n───┼───┼───\n")
+ }
+```
+
+Now we can run our application using scala-cli:
+
+```shell
+$ scala-cli run .
+Missing expected positional argument!
+
+Usage: sudokuSolver
+
+Solves sudokus passed as 81 chars string with . or 0 in place of empty cells
+
+Options and flags:
+ --help
+ Display this help text.
+
+$ scala-cli run . -- "483591267957268431621473895879132654164985372235647918792314586348756129516829743"
+483│591│267
+957│268│431
+621│473│895
+───┼───┼───
+879│132│654
+164│985│372
+235│647│918
+───┼───┼───
+792│314│586
+348│756│129
+516│829│743
+```
+
+Every argument that we will decide to support in the future will be documented in the `--help` output of our application.
+To solve the passed sudoku we must call the `solve` method on it and print both the success and failure cases to stdout and stderr, respectively.
+
+```scala
+/* src/Main.scala */
+import com.monovore.decline.*
+import cats.syntax.all.*
+import Sudoku.*
+
+val sudokuArgument: Opts[Sudoku] =
+ Opts.argument[String]("sudoku").mapValidated(Sudoku.from(_).toValidatedNel)
+
+object Main extends CommandApp(
+ name = "sudokuSolver",
+ header = "Solves sudokus passed as 81 chars string with . or 0 in place of empty cells",
+ main = sudokuArgument.map(sudoku =>
+ sudoku.solve.headOption.fold { // We will print the first solution only
+ System.err.println("Sudoku not solvable"); System.exit(1)
+ } {
+ s => System.out.println(s); System.exit(0)
+ }
+ )
+)
+```
+
+## 8. Packaging as an executable file
+
+It's time to use scala-cli to [package](https://scala-cli.virtuslab.org/docs/cookbooks/scala-package) our application. By default, scala-cli packages in a [lightweight format](https://scala-cli.virtuslab.org/docs/cookbooks/scala-package) that contains only your bytecode. To run the application, the `java` command needs to be available, and access to the internet, if dependencies need to be downloaded. Adding `//> using packaging.output "sudokuSolver"` to `project.scala` will let us control the filename of the produced executable file.
+
+```shell
+$ scala-cli package .
+Wrote /Users/toniogela/repo/sudoku/sudokuSolver, run it with
+ ./sudokuSolver
+
+$ ./sudokuSolver "....47......5.....9.483..15.19.7...4...3.9.21.3...5.7......8....78.2..3...1.5.4.."
+185│247│963
+763│591│248
+924│836│715
+───┼───┼───
+819│672│354
+457│389│621
+236│415│879
+───┼───┼───
+342│168│597
+578│924│136
+691│753│482
+
+$ ./sudokuSolver "foo"
+The sudoku string doesn't contain only digits
+
+Usage: sudokuSolver
+
+Solves sudokus passed as 81 chars string with . or 0 in place of empty cells
+
+Options and flags:
+ --help
+ Display this help text.
+```
+
+Running `sudokuSolver` on a fresh machine featuring only a Java installation will automagically download every dependency needed to run your code, plus macOS and Linux executables are portable between these two operating systems, maximising the "shareability" of your application package 😍.
+
+## 9. Benchmarking and Scala Native
+
+Now that we have a universal-ish binary that can run virtually anywhere there's a java installation, it's time for some benchmarking. To benchmark the time, our command line app takes to solve a specific sudoku we'll use [hyperfine](https://github.com/sharkdp/hyperfine), an awesome command-line benchmarking tool written in Rust.
+
+```shell
+$ SUDOKU="....47......5.....9.483..15.19.7...4...3.9.21.3...5.7......8....78.2..3...1.5.4.."
+
+$ hyperfine --warmup 20 -N "./sudokuSolver ${SUDOKU}"
+Benchmark 1:
+ Time (mean ± σ): 855.4 ms ± 8.7 ms [User: 1738.1 ms, System: 102.3 ms]
+ Range (min … max): 845.6 ms … 870.3 ms 10 runs
+```
+
+Running 20 warmup tests and avoiding spawning the command in a subshell (using `-N`) to reduce the number of statistical outliers, we get a mean resolution time of `855.4 ms`, which is a good, but not so good time.
+
+To get a considerable performance increase, we will leverage [Scala Native](https://scala-native.org/en/stable), an optimising ahead-of-time compiler and lightweight managed runtime specifically designed for Scala. To use libraries with Scala Native, they must be built specifically for it, and recently [several major Typelevel projects were published for it](https://typelevel.org/blog/2022/09/19/typelevel-native.html). Luckily for us, the list includes cats and decline, so we can seamlessly compile our application to native code.
+
+To achieve it, we can add a few directives to `project.scala`:
+- `//> using platform "scala-native"` to toggle the native compilation
+- `//> using nativeMode "release-full"` to choose a release mode optimised for performance
+- `//> using nativeGc "none"` to disable the garbage collector completely: this is an opinionated but safe choice since our app is a short-lived one
+
+Tweaking the `packaging.output` directive to rename the executable to `"sudokuNativeSolver"` and waiting for a while ("release-full" mode will significantly increase compilation times) will result in a native executable:
+
+```shell
+$ scala-cli package -f .
+Compiling project (Scala 3.2.1, Scala Native)
+Compiled project (Scala 3.2.1, Scala Native)
+[info] Linking (3057 ms)
+[info] Discovered 2011 classes and 13735 methods
+[info] Optimizing (release-full mode) (73201 ms)
+[info] Generating intermediate code (9415 ms)
+[info] Produced 1 files
+[info] Compiling to native code (110000 ms)
+[info] Total (196293 ms)
+Wrote /Users/toniogela/Downloads/sudoku/sudokuNativeSolver, run it with
+ ./sudokuNativeSolver
+
+$ SUDOKU="....47......5.....9.483..15.19.7...4...3.9.21.3...5.7......8....78.2..3...1.5.4.."
+
+$ ./sudokuNativeSolver $SUDOKU
+185│247│963
+763│591│248
+924│836│715
+───┼───┼───
+819│672│354
+457│389│621
+236│415│879
+───┼───┼───
+342│168│597
+578│924│136
+691│753│482
+```
+
+Now that we have two versions of the app, we can compare them using hyperfine again:
+
+```shell
+$ hyperfine --warmup 20 -N "./sudokuSolver $SUDOKU" "./sudokuNativeSolver $SUDOKU"
+Benchmark 1: ./sudokuSolver
+ Time (mean ± σ): 860.5 ms ± 17.9 ms [User: 1751.0 ms, System: 103.1 ms]
+ Range (min … max): 844.8 ms … 903.9 ms 10 runs
+
+Benchmark 2: ./sudokuNativeSolver
+ Time (mean ± σ): 122.5 ms ± 2.1 ms [User: 83.4 ms, System: 34.0 ms]
+ Range (min … max): 119.3 ms … 127.5 ms 24 runs
+
+Summary
+ './sudokuNativeSolver' ran 7.02 ± 0.19 times faster than './sudokuSolver'
+```
+
+As we can see, Scala Native annihilated the application startup time (there's no JVM to startup) and reduced the whole computing time altogether by seven times.
+
+Scala Native can be used to craft NGINX Unit server applications using [Snunit](https://github.com/lolgab/snunit), and as recently a [Cats-Effect single-threaded native runtime](https://github.com/armanbilge/epollcat) was published, it's possible to use [Http4s with Ember](https://github.com/ChristopherDavenport/scala-native-ember-example) to create single-threaded native servers!
+
+## 10. Conclusion
+
+In this article, we saw how to use [scala-cli](https://scala-cli.virtuslab.org/), [Scala Native](https://scala-native.org/en/stable/) and [decline](https://ben.kirw.in/decline/), a combination of tools that rocks when used to craft lightweight command line tools and much more. Despite not being a comprehensive guide to all their features, I hope this article will act as a starting point for ideas.
+
+Some ideas for further improvements:
+- Writing a `solve` stack-safe implementation
+- Adding a `--all` flag to print all the solutions
diff --git a/_posts/2023-01-30-optimizing-kafka-clients-a-hands-on-guide.md b/_posts/2023-01-30-optimizing-kafka-clients-a-hands-on-guide.md
new file mode 100644
index 000000000000..10b5e9cada4a
--- /dev/null
+++ b/_posts/2023-01-30-optimizing-kafka-clients-a-hands-on-guide.md
@@ -0,0 +1,485 @@
+---
+title: "Optimizing Kafka Clients: A Hands-On Guide"
+date: 2023-01-22
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kotlin, kafka, streaming]
+excerpt: "Tips on how to make Kafka clients run blazing fast, with code examples."
+attribution: Squirrel Cartoon Vectors by Vecteezy
+---
+
+_This article is brought to you by [Giannis Polyzos](https://github.com/polyzos). Giannis is a proud alumnus of Rock the JVM, working as a Solutions Architect with a focus on Event Streaming and Stream Processing Systems._
+
+_Enter Giannis:_
+
+## 1. Introduction
+
+Apache Kafka is a well-known event streaming platform used in many organizations worldwide.
+It is used as the backbone of many data infrastructures, thus it's important to understand how to use it efficiently.
+The focus of this article is to provide a better understanding of how Kafka works under the hood to better design and tune your client applications.
+
+Since we will discuss **how things work**, this article assumes some basic familiarity with Kafka, i.e.:
+- Understanding Kafka on a high level
+- Experience with the Java Client API for creating Producers and Consumers
+- Some familiarity with Docker will be helpful (but not required)
+
+
+
+We will use a file ingestion data pipeline for clickstream data as an example to cover the following:
+1. Ingest click stream data from the filesystem into Kafka
+2. Explain how Kafka producers work, configurations and tuning for throughput / latency
+3. How Kafka consumers work, configurations and scaling the consuming side
+4. Caveats with consumer offsets and different approaches for handling them
+
+The relevant e-commerce datasets can be found [here](https://www.kaggle.com/datasets/mkechinov/ecommerce-behavior-data-from-multi-category-store).
+The code samples are written in Kotlin, but the implementation should be easy to port in Java or Scala.
+You can find the source code on Github [here](https://github.com/polyzos/kafka-streaming-ledger).
+
+> _To make the most out of this article, I'd recommend cloning the GitHub repo and following the code snippets there, so you can run them easily, instead of reproducing things yourself from scratch, unless you're a Kafka expert already._
+
+So, let us dive right in.
+
+## 2. Environment Setup
+
+First, we want to have a Kafka Cluster up and running.
+Make sure you have [docker compose](https://docs.docker.com/compose/) installed on your machine, as we will use the following `docker-compose.yaml` file to set up a 3-node Kafka Cluster.
+
+```shell
+version: "3.7"
+services:
+ zookeeper:
+ image: bitnami/zookeeper:3.8.0
+ ports:
+ - "2181:2181"
+ volumes:
+ - ./logs/zookeeper:/bitnami/zookeeper
+ environment:
+ ALLOW_ANONYMOUS_LOGIN: "yes"
+ kafka0:
+ image: bitnami/kafka:3.3.1
+ ports:
+ - "9092:9092"
+ volumes:
+ - ./logs/kafka0:/bitnami/kafka
+ environment:
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_ADVERTISED_PORT: 9092
+ KAFKA_ADVERTISED_HOST_NAME: kafka0
+ KAFKA_LISTENERS: >-
+ INTERNAL://:29092,EXTERNAL://:9092
+ KAFKA_ADVERTISED_LISTENERS: >-
+ INTERNAL://kafka0:29092,EXTERNAL://localhost:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
+ INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
+ depends_on:
+ - zookeeper
+ kafka1:
+ image: bitnami/kafka:3.3.1
+ ports:
+ - "9093:9093"
+ volumes:
+ - ./logs/kafka1:/bitnami/kafka
+ environment:
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_LISTENERS: >-
+ INTERNAL://:29092,EXTERNAL://:9093
+ KAFKA_ADVERTISED_LISTENERS: >-
+ INTERNAL://kafka1:29092,EXTERNAL://localhost:9093
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
+ INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
+ depends_on:
+ - zookeeper
+ kafka2:
+ image: bitnami/kafka:3.3.1
+ ports:
+ - "9094:9094"
+ volumes:
+ - ./logs/kafka2:/bitnami/kafka
+ environment:
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_LISTENERS: >-
+ INTERNAL://:29092,EXTERNAL://:9094
+ KAFKA_ADVERTISED_LISTENERS: >-
+ INTERNAL://kafka2:29092,EXTERNAL://localhost:9094
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
+ INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
+ depends_on:
+ - zookeeper
+```
+
+All you have to do is navigate to the root folder of the project where the `docker-compose.yaml` file is located and run
+
+```shell
+docker-compose up
+```
+
+This command will start a 3-node Kafka Cluster.
+
+> _Note: You might want to increase your docker resources (I'm running with 6GB RAM) to make sure you don't run into any issues._
+
+Wait a bit for the cluster to start, then we will need to create our topic to store our clickstream events. For that we will create a topic with 5 partitions and a replication factor of 3 (leader + 2 replicas) using the following command:
+
+```shell
+bin/kafka-topics.sh --create \
+ --topic ecommerce.events \
+ --replication-factor 3 \
+ --partitions 5 \
+ --bootstrap-server localhost:9092
+```
+
+Verify the topic was created successfully:
+```shell
+bin/kafka-topics.sh --bootstrap-server localhost:9092 --describe ecommerce.events
+
+Topic: ecommerce.events TopicId: oMhOjOKcQZaoPp_8Xc27lQ PartitionCount: 5 ReplicationFactor: 3 Configs:
+ Topic: ecommerce.events Partition: 0 Leader: 1001 Replicas: 1001,1002,1003 Isr: 1001,1002,1003
+ Topic: ecommerce.events Partition: 1 Leader: 1003 Replicas: 1003,1001,1002 Isr: 1003,1001,1002
+ Topic: ecommerce.events Partition: 2 Leader: 1002 Replicas: 1002,1003,1001 Isr: 1002,1003,1001
+ Topic: ecommerce.events Partition: 3 Leader: 1001 Replicas: 1001,1003,1002 Isr: 1001,1003,1002
+ Topic: ecommerce.events Partition: 4 Leader: 1003 Replicas: 1003,1002,1001 Isr: 1003,1002,1001
+```
+
+## 3. Show me the Code 👀
+
+The producer will send some events to Kafka.
+
+The data model for the click events should look similar to the following payload.
+
+
+Creating Kafka Producers is pretty straightforward, the important part is creating and sending records and the produce method should look similar to the following:
+```kotlin
+fun produce(topic: String, key: K, value: V) {
+ ProducerRecord(topic, key, value)
+ .also { record ->
+ producer.send(record) { _, exception ->
+ exception?.let {
+ logger.error { "Error while producing: $exception" }
+ } ?: kotlin.run {
+ counter.incrementAndGet()
+ if (counter.get() % 100000 == 0) {
+ logger.info { "Total messages sent so far ${counter.get()}." }
+ }
+ }
+ }
+ }
+}
+
+producerResource.produce(KafkaConfig.EVENTS_TOPIC, event.userid, event)
+```
+For every event we create a **ProducerRecord** object and specify the `topic`, the `key` (here we partition on the userId),
+and finally the event payload as the `value`.
+
+The `send()` method is asynchronous, so we specify a callback that gets triggered when we receive a result back.
+
+If the message was successfully written to Kafka we print the metadata, otherwise if an exception is returned we log it.
+
+_But what actually happens when the `send()` method is called?_
+
+
+
+Kafka does a lot of things under the hood when the `send()` method is invoked, so let’s outline them below:
+
+1. The message is serialized using the specified serializer.
+2. The partitioner determines which partition the message should be routed to.
+3. Internally Kafka keeps message buffers; we have one buffer for each partition and each buffer can hold many batches of messages grouped for each partition.
+4. Finally, the I/O threads pick up these batches and sent them over to the brokers.
+At this point, our messages are in-flight from the client to the brokers. The brokers have sent/receive network buffers for the network threads to pick up the messages and hand them over to some IO thread to actually persist them on disk.
+5. On the leader broker, the messages are written on disk and sent to the followers for replication. One thing to note here is that the messages are first written on the PageCache and periodically are flushed on disk.
+(_Note:_ PageCache to disk is an extreme case for message loss, but still you might want to be aware of that)
+6. The followers (in-sync replicas) store and sent an acknowledgment back they have replicated the message.
+7. A `RecordMetadata` response is sent back to the client.
+8. If a failure occurred without receiving an ACK, we check if message retry is enabled; if so, we need to resend it.
+9. The client receives the response.
+
+## 4. Tradeoffs between Latency and Throughput
+
+In distributed systems, most things come with tradeoffs, and it’s up to the developer to find that "sweet spot" between different tradeoffs; thus it’s important to understand how things work.
+
+One important aspect might be tuning between throughput and latency. Some key configurations to that are `batch.size` and `linger.ms`. These configs work as follows: the producer will batch together records, and when the batch is full, it will send it whole. Otherwise, it will wait at most `linger.ms` to enqueue a new item, and if that time is expired, it will send a (partially full) batch.
+
+Having a small `batch.size` and also `linger` set to 0 can reduce latency and process messages as soon as possible — but it might reduce throughput. Configuring for low latency is also useful for slow produce rate scenarios. Having fewer records accumulated than the specified `batch.size` will result in the client waiting `linger.ms` for more records to arrive.
+
+On the other hand, a larger `batch.size` might use more memory (as we will allocate buffers for the specified batch size) but it will increase the throughput. Other configuration options like `compression.type`, `max.in.flight.requests.per.connection`, and `max.request.size` can help here.
+
+
+
+Let’s better illustrate this with an example.
+
+Our event data is stored in CSV files that we want to ingest into Kafka, and since it is not real-time data ingestion, we don’t really care about latency here, but having a good throughput so that we can ingest them fast.
+
+Using the default configurations ingesting 5.000.000 messages takes around 119 seconds.
+
+```shell
+13:17:22.885 INFO [kafka-producer-network-thread | producer-1] io.ipolyzos.utils.LoggingUtils - Total messages sent so far 4800000.
+13:17:25.323 INFO [kafka-producer-network-thread | producer-1] io.ipolyzos.utils.LoggingUtils - Total messages sent so far 4900000.
+13:17:27.960 INFO [kafka-producer-network-thread | producer-1] io.ipolyzos.utils.LoggingUtils - Total messages sent so far 5000000.
+13:17:27.967 INFO [main] io.ipolyzos.producers.ECommerceProducer - Total time '119' seconds
+13:17:27.968 INFO [main] io.ipolyzos.utils.LoggingUtils - Total Event records sent: '5000000'
+13:17:27.968 INFO [main] io.ipolyzos.utils.LoggingUtils - Closing Producers ...
+```
+Setting `batch.size` to 64Kb (16 is the default), linger.ms greater than 0 and finally `compression.type` to gzip
+```kotlin
+ properties[ProducerConfig.BATCH_SIZE_CONFIG] = "64000"
+ properties[ProducerConfig.LINGER_MS_CONFIG] = "20"
+ properties[ProducerConfig.COMPRESSION_TYPE_CONFIG] = "gzip"
+```
+Has the following impact on the ingestion time.
+
+```shell
+13:18:34.377 INFO [kafka-producer-network-thread | producer-1] io.ipolyzos.utils.LoggingUtils - Total messages sent so far 4800000.
+13:18:35.280 INFO [kafka-producer-network-thread | producer-1] io.ipolyzos.utils.LoggingUtils - Total messages sent so far 4900000.
+13:18:35.980 INFO [kafka-producer-network-thread | producer-1] io.ipolyzos.utils.LoggingUtils - Total messages sent so far 5000000.
+13:18:35.983 INFO [main] io.ipolyzos.producers.ECommerceProducer - Total time '36' seconds
+13:18:35.984 INFO [main] io.ipolyzos.utils.LoggingUtils - Total Event records sent: '5000000'
+13:18:35.984 INFO [main] io.ipolyzos.utils.LoggingUtils - Closing Producers ...
+```
+
+From around 119 seconds, the time dropped to 36 seconds. In both cases `ack=1`.
+I will leave it up to you to experiment and try different configuration options to see how they better come in handy based on your use case.
+You might also want to test against a real cluster to test the networking in place. For example running this similar example on a real cluster takes around _184 seconds_ to ingest 1000000 messages and when adding the configurations changes drops down to _18 seconds_.
+
+If you are concerned with exactly-once semantics, set `enable.idempotency` to true, which will also result in ACKs set to all.
+
+## 5. Kafka Consumers: Switching to the Other Side of the Wall
+
+Up to this point, we have ingested clickstream events into Kafka, so let's see what reading those events looks like.
+
+A typical Kafka consumer loop should look similar to the following snippet:
+```kotlin
+private fun consume(topic: String) {
+ consumer.subscribe(listOf(topic))
+ try {
+ while (true) {
+ val records: ConsumerRecords = consumer.poll(Duration.ofMillis(100))
+ records.forEach { record ->
+ // simulate the consumers doing some work
+ Thread.sleep(20)
+ logger.info { record }
+ }
+ logger.info { "[$consumerName] Records in batch: ${records.count()} - Total Consumer Processed: ${counter.get()} - Total Group Processed: ${consumePartitionMsgCount.get()}" }
+ }
+ } catch (e: WakeupException) {
+ logger.info("[$consumerName] Received Wake up exception!")
+ } catch (e: Exception) {
+ logger.warn("[$consumerName] Unexpected exception: {}", e.message)
+ } finally {
+ consumer.close()
+ logger.info("[$consumerName] The consumer is now gracefully closed.")
+ }
+ }
+```
+
+Let’s try to better understand what happens here. The following diagram provides a more detailed explanation.
+
+
+Kafka uses a pull-based model for data fetching. At the "heart of the consumer" sits the poll loop. The poll loop is important for two reasons:
+1. It is responsible for fetching data (providing **ConsumerRecords**) for the consumer to process and
+2. Sends heartbeats and coordinates the consumers so the consumer group knows the available consumers and if a rebalancing needs to take place.
+
+The consuming applications maintain TCP connections with the brokers and sent fetch requests to fetch data. The data is cached and periodically returned from the _poll()_ method. When data is returned from the _poll()_ method the actual processing takes place and once it’s finished more data is requested and so on.
+
+What’s important to note here (and we will dive deeper into it in the next part) is committing message offsets. This is Kafka’s way of knowing that a message has been fetched and processed successfully. By default, offsets are committed automatically at regular intervals.
+
+The amount of data - how much it is going to be fetched, when more data needs to be requested etc. are dictated by configuration options like, `fetch.min.bytes`, `max.partition.fetch.bytes`, `fetch.max.bytes`, `fetch.max.wait.ms`. You might think that the default options might be ok for you, but it’s important to test them out and think through your use case carefully.
+
+To make this more clear let’s assume that you fetch 500 records from the `poll()` loop to process, but the processing for some reason takes too long for each message. The config `max.poll.interval.ms` dictates the maximum time a consumer can be idle before fetching more records; i.e. calling the poll method and if this threshold is reached the consumer is considered `lost` and a rebalance will be triggered — although our application was just slow on processing.
+
+So decreasing the number of records the `poll()` loop should return and/or better tuning some configurations like `heartbeat.interval.ms` and `session.timeout.ms` used for consumer group coordination might be reasonable in this case.
+
+## 6. Running the Consumer
+
+At this point, I will start one consuming instance on my `ecommerce.events` topic. Remember that this topic consists of 5 partitions.
+I will execute against my Kafka cluster, using the default consumer configuration options and my goal is to see how long it takes for a consumer to read 10000 messages from the topic, assuming a 20ms processing time per message.
+
+```shell
+12:37:13.362 INFO [main] io.ipolyzos.utils.LoggingUtils - [Consumer-1] Records in batch: 500 - Elapsed Time: 226 seconds - Total Consumer Processed: 9500 - Total Group Processed: 9500
+12:37:25.039 INFO [main] io.ipolyzos.Extensions - Batch Contains: 500 records from 1 partitions.
+12:37:25.040 INFO [main] io.ipolyzos.Extensions -
++--------+------------------+-----------+---------------+-----------+----------------------------------------------+---------------+-------------------------------------------------+-------------+
+| Offset | Topic | Partition | Timestamp | Key | Value | TimestampType | Teaders | LeaderEpoch |
++--------+------------------+-----------+---------------+-----------+----------------------------------------------+---------------+-------------------------------------------------+-------------+
+| 9999 | ecommerce.events | 4 | 1674469663415 | 513376382 | {eventTime=1572569126000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 9998 | ecommerce.events | 4 | 1674469663415 | 529836227 | {eventTime=1572569124000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 9997 | ecommerce.events | 4 | 1674469663415 | 513231964 | {eventTime=1572569124000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 9996 | ecommerce.events | 4 | 1674469663415 | 564378632 | {eventTime=1572569124000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 9995 | ecommerce.events | 4 | 1674469663415 | 548411752 | {eventTime=1572569124000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
++--------+------------------+-----------+---------------+-----------+----------------------------------------------+---------------+-------------------------------------------------+-------------+
+12:37:25.041 INFO [main] io.ipolyzos.utils.LoggingUtils - [Consumer-1] Records in batch: 500 - Elapsed Time: 237 seconds - Total Consumer Processed: 10000 - Total Group Processed: 10000
+12:37:27.509 INFO [Thread-0] io.ipolyzos.utils.LoggingUtils - [Consumer-1] Detected a shutdown, let's exit by calling consumer.wakeup()...
+
+```
+We can see that it takes a single consumer around 4 minutes for this kind of processing. So how can we do better?
+
+## 7. Scaling the Consuming Side
+
+Consumer Groups are Kafka’s way of sharing the work between different consumers and also the level of parallelism. The highest level of parallelism you can achieve with Kafka is having one consumer consuming from each partition of a topic.
+
+### 7.1. #Partitions > #Consumers
+
+In the scenario, the available partitions will be shared equally among the available consumers of the group and each consumer will have ownership of those partitions.
+
+
+
+### 7.2. #Partitions = #Consumers
+
+
+
+When the partition number is equal to the available consumers each consumer will be reading from exactly one partition. In this scenario, we also reach the maximum parallelism we can achieve on a particular topic.
+
+### 7.3. #Partitions < #Consumers
+
+
+This scenario is similar to the previous one, only now we will have one consumer running but stays idle. On the one hand, this means we waste resources, but we can also use this consumer as a _failover_ in case another one in the group goes down.
+
+When a consumer goes down or similarly a new one joins the group, Kafka will have to trigger a rebalance. This means that partitions need to be revoked and reassigned to the available consumers in the group.
+
+Let’s run again our previous example — consuming 10k messages — but this time having 5 consumers in our consumer group. I will be creating 5 consuming instances from within a single JVM (using Kotlin [coroutines](/kotlin-coroutines-101), but you can easily re-adjust the code (found [here](https://github.com/polyzos/kafka-streaming-ledger/blob/main/src/main/kotlin/io/ipolyzos/consumers/PerPartitionConsumer.kt)) and just start multiple JVMs.
+```shell
+12:39:53.233 INFO [DefaultDispatcher-worker-1] io.ipolyzos.Extensions -
++--------+------------------+-----------+---------------+-----------+----------------------------------------------+---------------+-------------------------------------------------+-------------+
+| Offset | Topic | Partition | Timestamp | Key | Value | TimestampType | Teaders | LeaderEpoch |
++--------+------------------+-----------+---------------+-----------+----------------------------------------------+---------------+-------------------------------------------------+-------------+
+| 1999 | ecommerce.events | 0 | 1674469663077 | 512801708 | {eventTime=1572562526000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 1998 | ecommerce.events | 0 | 1674469663077 | 548190452 | {eventTime=1572562525000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 1997 | ecommerce.events | 0 | 1674469663077 | 517445703 | {eventTime=1572562520000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 1996 | ecommerce.events | 0 | 1674469663077 | 512797937 | {eventTime=1572562519000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
+| 1995 | ecommerce.events | 0 | 1674469663077 | 562837353 | {eventTime=1572562519000, eventType=view ... | CreateTime | RecordHeaders(headers = [], isReadOnly = false) | Optional[1] |
++--------+------------------+-----------+---------------+-----------+----------------------------------------------+---------------+-------------------------------------------------+-------------+
+12:39:53.233 INFO [DefaultDispatcher-worker-1] io.ipolyzos.utils.LoggingUtils - [Consumer-2] Records in batch: 500 - Elapsed Time: 47 seconds - Total Consumer Processed: 2000 - Total Group Processed: 10000
+12:39:55.555 INFO [Thread-10] io.ipolyzos.utils.LoggingUtils - [Consumer-3] Detected a shutdown, let's exit by calling consumer.wakeup()...
+```
+
+As expected we can see that the consumption time dropped to less than a minute time.
+
+But if Kafka’s maximum level of parallelism is one consumer per partition, does this mean we hit the scaling limit? Let’s see how to tackle this next.
+
+### 7.4. The Parallel Consumer Pattern
+
+Up to this point, we might have two questions in mind:
+1. If #partitions = #consumers in the consumer group, how can I scale even further if needed? It’s not always easy to calculate the number of partitions beforehand and/or I might need to account for sudden spikes.
+2. How can I minimize rebalancing time?
+
+One solution to this can be the parallel consumer pattern. You can have consumers in your group consuming from one or more partitions of the topic, but then they propagate the actual processing to other threads.
+
+One such implementation can be found [here](https://github.com/confluentinc/parallel-consumer).
+
+It provides three ordering guarantees — _Unordered_, _Keyed_ and _Partition_.
+1. _Unordered_ — provides no guarantees
+2. _Key_ — guarantees ordering per key BUT with the caveat that the keyspace needs to be quite large, otherwise you might not observe much performance improvement.
+3. _Partition—Only_ one message will be processed per partition at any time.
+
+Along with that it also provides different ways for committing offset. This is a pretty nice library you might want to look at.
+
+
+
+Going once more back to our example to answer the question — how can we break the scaling limit? We will be using the parallel consumer pattern — you can find the code [here](https://github.com/polyzos/kafka-streaming-ledger/blob/main/src/main/kotlin/io/ipolyzos/consumers/ParallelScalingConsumer.kt).
+Using one _parallel consumer instance_ on our _5-partition_ topic, specifying a _Key Ordering_, and using a parallelism of _100 threads_
+```shell
+12:48:42.752 INFO [pc-pool-1-thread-87] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9970 - Total Group Processed: 9970
+12:48:42.752 INFO [pc-pool-1-thread-98] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9970 - Total Group Processed: 9970
+12:48:42.758 INFO [pc-pool-1-thread-26] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9975 - Total Group Processed: 9975
+12:48:42.761 INFO [pc-pool-1-thread-72] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9980 - Total Group Processed: 9980
+12:48:42.761 INFO [pc-pool-1-thread-75] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9985 - Total Group Processed: 9985
+12:48:42.761 INFO [pc-pool-1-thread-21] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9990 - Total Group Processed: 9990
+12:48:42.765 INFO [pc-pool-1-thread-13] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 9995 - Total Group Processed: 9995
+12:48:42.765 INFO [pc-pool-1-thread-56] io.ipolyzos.utils.LoggingUtils - [Consumer-0] Records in batch: 5 - Elapsed Time: 2 seconds - Total Consumer Processed: 10000 - Total Group Processed: 10000
+```
+
+Makes the consuming and processing time of **10k messages** take as much as **2 seconds**.
+
+> _Notice from the logs how different batches are processed on different threads on the same consumer instance._
+
+and if we use _5 parallel consumer instances_, it takes just a few milliseconds.
+```shell
+12:53:23.800 INFO [pc-pool-2-thread-85] io.ipolyzos.utils.LoggingUtils - [Consumer-1] Records in batch: 5 - Elapsed Time: 672 milliseconds - Total Consumer Processed: 2031 - Total Group Processed: 10015
+12:53:23.800 INFO [pc-pool-4-thread-11] io.ipolyzos.utils.LoggingUtils - [Consumer-3] Records in batch: 5 - Elapsed Time: 683 milliseconds - Total Consumer Processed: 1865 - Total Group Processed: 10015
+12:53:23.800 INFO [pc-pool-5-thread-80] io.ipolyzos.utils.LoggingUtils - [Consumer-5] Records in batch: 5 - Elapsed Time: 670 milliseconds - Total Consumer Processed: 2062 - Total Group Processed: 10020
+12:53:23.801 INFO [pc-pool-6-thread-6] io.ipolyzos.utils.LoggingUtils - [Consumer-4] Records in batch: 5 - Elapsed Time: 654 milliseconds - Total Consumer Processed: 2018 - Total Group Processed: 10025
+12:53:23.801 INFO [pc-pool-4-thread-41] io.ipolyzos.utils.LoggingUtils - [Consumer-3] Records in batch: 5 - Elapsed Time: 684 milliseconds - Total Consumer Processed: 1870 - Total Group Processed: 10030
+12:53:23.801 INFO [pc-pool-3-thread-8] io.ipolyzos.utils.LoggingUtils - [Consumer-2] Records in batch: 5 - Elapsed Time: 672 milliseconds - Total Consumer Processed: 2054 - Total Group Processed: 10035
+12:53:23.801 INFO [pc-pool-5-thread-89] io.ipolyzos.utils.LoggingUtils - [Consumer-5] Records in batch: 5 - Elapsed Time: 671 milliseconds - Total Consumer Processed: 2067 - Total Group Processed: 10040
+12:53:23.801 INFO [pc-pool-5-thread-66] io.ipolyzos.utils.LoggingUtils - [Consumer-5] Records in batch: 5 - Elapsed Time: 671 milliseconds - Total Consumer Processed: 2072 - Total Group Processed: 10045
+12:
+```
+So basically we have accomplished getting our processing time from ~ minutes down to seconds.
+
+> _Notice in the screenshot how different batches are processed on different threads on different consumer instances._
+
+## 8. Offsets and How to Handle Them
+
+Up to this point, we have seen the whole message lifecycle in Kafka — PPC (Produce, Persist, Consume)
+
+One thing really important though — especially when you need to trust your system provides the best guarantees when processing each message exactly once — is committing offsets.
+
+Fetching messages from Kafka, processing them and marking them as processed, by actually providing such guarantees has a few pitfalls and is not provided out of the box.
+
+This is what we will see next, i.e. what do I need to take into account to get the best possible exactly-once processing guarantees out of my applications?
+
+We will take a look at a few scenarios for committing offsets and what caveats each approach might have.
+
+### 8.1. Committing Offsets Automatically
+This is the default behavior with enable.auto.commit set to true. The caveat here is that the message is consumed and the offsets will be committed periodically, BUT this doesn’t mean the message has been successfully processed. If the message fails for some reason, its offset might have been committed and as far as Kafka is concerned that message has been processed successfully.
+
+### 8.2. Committing Offsets Manually
+Setting `enable.auto.commit` to false takes Kafka consumers out of the "autopilot mode" and it’s up to the application to commit the offsets. This can be achieved by using the `commitSync()` or `commitAsync()` methods on the consumer API.
+
+When committing offsets manually we can do so either when the whole batch returned from the `poll()` method has finished processing in which case all the offsets up to the highest one will be committed, or we might want to commit after each individual message is done with it’s processing for even stronger guarantees.
+
+#### 8.2.1. Commit/Message
+
+```kotlin
+if (perMessageCommit) {
+ logger.info { "Processed Message: ${record.value()} with offset: ${record.partition()}:${record.offset()}" }
+ val commitEntry: Map =
+ mapOf(TopicPartition(record.topic(), record.partition()) to OffsetAndMetadata(record.offset()))
+
+ consumer.commitSync(commitEntry)
+ logger.info { "Committed offset: ${record.partition()}:${record.offset()}" }
+}
+```
+
+#### 8.2.2. Commit/Batch
+
+```kotlin
+// Process all the records
+records.forEach { record ->
+ // simulate the consumers doing some work
+ Thread.sleep(20)
+ logger.info { "Processing Record: ${record.value()}" }
+}
+// Commit up to the highest offset
+val maxOffset: Long = record.maxOfOrNull { it.offset() }!!
+conssumer.commitAsync { topicPartition, offsetMetadata ->
+ logger.info { "Processed up to offset: $maxOffset - Committed: $topicPartition and $offsetMetadata" }
+
+}
+```
+This gives us control over how message offsets are committed, and we can trust that we will wait for the actual processing to finish before committing the offset.
+
+For those who want to account for (or at least try to) _every unhappy_ path there is also the possibility that things fail in the commit process itself. In this case the message will be reprocessed
+
+### 8.3. Idempotency with External Storage
+
+You can use an external data store and keep track of the offsets there (for example like Apache Cassandra).
+
+Consuming messages and using something like a transaction for both processing the message, as well as committing the offsets will guarantee that either both will succeed or fail and thus idempotency is ensured.
+
+One thing to note here is that offsets are now stored in an external datastore. When starting a new consumer or a rebalancing takes place you need to make sure your consumer fetches the offsets from the external datastore.
+
+One way to achieve this can be adding a `ConsumerRebalanceListener` and when `onPartitionsRevoked` and `onPartitionsAssigned` methods are called store (commit) or retrieve the offsets from the external datastore.
+
+
+## 9. Wrapping Up
+
+As takeaways:
+- Don't rely on the default configurations; Try to fine-tune between throughput and latency
+- Partitions play an important role in scaling Kafka.
+- The Parallel-Consumer pattern can further help you scale your applications.
+- Consuming a message is different from actually processing it successfully
+- Auto-committing offsets can have a negative impact on your application guarantees
diff --git a/_posts/2023-02-06-flink-sql-introduction.md b/_posts/2023-02-06-flink-sql-introduction.md
new file mode 100644
index 000000000000..51d2d51cb3fa
--- /dev/null
+++ b/_posts/2023-02-06-flink-sql-introduction.md
@@ -0,0 +1,719 @@
+---
+title: "Streaming SQL with Apache Flink: A Gentle Introduction"
+date: 2023-02-06
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [flink, kafka, streaming]
+excerpt: "A hands-on guide to Flink SQL for data streaming with familiar tools."
+---
+
+_This article is brought to you by [Giannis Polyzos](https://github.com/polyzos). Giannis is a proud alumnus of Rock the JVM, working as a Solutions Architect with a focus on Event Streaming and Stream Processing Systems._
+
+_Enter Giannis:_
+
+Flink SQL is a powerful high level API for running queries on streaming (and batch) datasets. In this article we will see:
+1. Why it's powerful and how it helps democratize Stream Processing and Analytics
+2. Understand basic concepts around Streaming and Flink SQL
+3. Setup Kafka and Flink Clusters and get started with Flink SQL
+4. Understand different kinds of Processing Operators and Functions
+5. Different ways of running Flink SQL Queries
+
+> _Daniel here: If you need to learn the fundamentals of streaming with Flink, check out the long-form [Flink course](https://rockthejvm.com/p/flink) here at Rock the JVM._
+
+## 1. Streaming (and Batch) SQL
+### 1.1 Unified Batch and Streaming
+
+When we hear about SQL (referenced as batch SQL here) we think of the following tabular format you typically find in RDBMS,
+on which we operate and run computations - from simple projections (like SELECT and Filter), to Aggregations to Windowed Functions.
+
+
+
+Batch SQL Queries operate on static data, i.e. on data stored on disk, already available and the results are considered complete.
+
+How can Tables relate with Streams?
+
+Let's think of a data stream now
+
+
+
+A stream is basically an *unbounded* dataset of incoming events, i.e. it has no end.
+In the heart of a stream is the *append-only log*, i.e. each incoming events can be considered as a *row* that gets appended at the end of the log - similar to a database table.
+
+In practice if we follow this mental model we can think of a stream as a collection of snapshots of bounded datasets.
+
+
+
+This is what is enables Unified Batch and Streaming Architecture and allows the use of a single API - like Flink SQL - to handle both Batch and Streaming data; no underlying code changes needed.
+
+### 1.2 Streaming SQL Semantics
+
+The rules are as follows:
+- The Input tables are constantly changing and possibly unbounded
+ - *Append Only Streams:* Keeps all the history in the stream. Every new event is an insert operation in the append-only log
+ - *Changelog Streams:* Keeps the most recent value (for some key).
+- Query results are never final, continuously updated, and potentially unbounded
+
+
+
+One the left side we have our append-log (or a collection of bounded datasets as we discussed above) and we run a *Streaming SQL Query* on that table.
+
+As we keep ingesting new events, they get appended as new rows to the log. These events yield changes, which results in the output table being continuously updated.
+
+This is called a *Dynamic Table*.
+
+## 2. Flink SQL Logical Components
+
+
+
+Flink consists of *catalogs* that hold metadata for databases, tables, functions and views.
+
+A catalog can be non-persisted (In Memory Catalog) or persistent backed by an external system like the PostgresCatalog, the PulsarCatalog and the HiveCatalog.
+
+For In Memory catalogs all metadata will be available only for the lifetime of the session.
+
+In contrast, catalogs like PostgresCatalog enables users to connect the two systems and then Flink
+automatically references existing metadata by mapping them to it’s corresponding metadata.
+
+For example, Flink can map Postgres tables to its own table automatically, and users don’t have to manually re-writing DDLs in Flink SQL.
+
+Within the catalogs you create databases and tables in these databases.
+
+When creating a table it's full table name identifier is: *..* and when a catalogs and/or database is not specified
+the default ones are used.
+
+## 3. Environment Setup
+As a warmup exercise let's start the Flink SQL CLI to run a few commands; but first we need to have a Flink and a Kafka Cluster up and running.
+
+Make sure you have [docker compose](https://docs.docker.com/compose/) installed on your machine, as we will use the following `docker-compose.yaml` file to set up the required clusters.
+
+```yaml
+version: "3.7"
+services:
+ zookeeper:
+ image: bitnami/zookeeper:3.8.0
+ ports:
+ - "2181:2181"
+ environment:
+ ALLOW_ANONYMOUS_LOGIN: "yes"
+ kafka:
+ image: bitnami/kafka:3.3.1
+ ports:
+ - "9092:9092"
+ environment:
+ KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+ ALLOW_PLAINTEXT_LISTENER: "yes"
+ KAFKA_ADVERTISED_PORT: 9092
+ KAFKA_ADVERTISED_HOST_NAME: kafka
+ KAFKA_LISTENERS: >-
+ INTERNAL://:29092,EXTERNAL://:9092
+ KAFKA_ADVERTISED_LISTENERS: >-
+ INTERNAL://kafka:29092,EXTERNAL://localhost:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
+ INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
+ depends_on:
+ - zookeeper
+ jobmanager:
+ build: .
+ container_name: jobmanager
+ ports:
+ - "8081:8081"
+ command: jobmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: jobmanager
+ taskmanager:
+ build: .
+ depends_on:
+ - jobmanager
+ command: taskmanager
+ deploy:
+ replicas: 1
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: jobmanager
+ taskmanager.numberOfTaskSlots: 1
+```
+
+### 3.1 The Flink SQL Client
+Run `docker-compose up`, wait for a few seconds and your clusters should be up and running.
+
+Let's start the Flink SQL CLI by running `docker exec -it jobmanager ./bin/sql-client.sh` and then execute the following commands as a warmup with the sql client:
+
+```shell
+Flink SQL> SHOW CATALOGS;
++-----------------+
+| catalog name |
++-----------------+
+| default_catalog |
++-----------------+
+1 row in set
+
+// Create a new Database
+Flink SQL> CREATE DATABASE mydbl
+
+Flink SQL> SHOW DATABASES;
++------------------+
+| database name |
++------------------+
+| default_database |
+| mydbl |
++------------------+
+2 rows in set
+
+Flink SQL> use mydbl;
+[INFO] Execute statement succeed.
+
+// currently we have no tables
+Flink SQL> SHOW TABLES;
+Empty set
+
+// a truncated output of some available functions.
+Flink SQL> SHOW FUNCTIONS;
++-------------------------------+
+| function name |
++-------------------------------+
+| AGG_DECIMAL_MINUS |
+| AGG_DECIMAL_PLUS |
+| ARRAY_CONTAINS |
+| COALESCE |
+| CURRENT_WATERMARK |
+| GREATEST |
+| IFNULL |
+| IS_JSON |
+| JSON_ARRAY |
+| JSON_ARRAYAGG_ABSENT_ON_NULL |
+| JSON_ARRAYAGG_NULL_ON_NULL |
+| JSON_EXISTS |
+| JSON_OBJECT |
+| JSON_OBJECTAGG_ABSENT_ON_NULL |
+| JSON_OBJECTAGG_NULL_ON_NULL |
+| JSON_QUERY |
+| JSON_STRING |
+| JSON_VALUE |
+| LEAST |
+| SOURCE_WATERMARK |
+
+```
+
+It's time now to get into some interesting stuff, but before that let's also create the Kafka topics we will be using for our examples.
+
+```shell
+docker exec -it depths-of-flink-kafka-1 kafka-topics.sh --create \
+ --topic sensor.info \
+ --partitions 1 \
+ --config cleanup.policy=compact \
+ --bootstrap-server localhost:9092
+
+docker exec -it depths-of-flink-kafka-1 kafka-topics.sh --create \
+ --topic sensor.readings \
+ --partitions 3 \
+ --bootstrap-server localhost:9092
+
+docker exec -it depths-of-flink-kafka-1 kafka-topics.sh --bootstrap-server localhost:9092 --describe
+
+------- Output -------
+Topic: sensor.info TopicId: zFY47WiRS721XIUik2nRBg PartitionCount: 1 ReplicationFactor: 1 Configs: cleanup.policy=compact
+ Topic: sensor.info Partition: 0 Leader: 1001 Replicas: 1001 Isr: 1001
+
+Topic: sensor.readings TopicId: HGvGHOeKQQCxG3cly2R7Lw PartitionCount: 3 ReplicationFactor: 1 Configs:
+ Topic: sensor.readings Partition: 0 Leader: 1001 Replicas: 1001 Isr: 1001
+ Topic: sensor.readings Partition: 1 Leader: 1001 Replicas: 1001 Isr: 1001
+ Topic: sensor.readings Partition: 2 Leader: 1001 Replicas: 1001 Isr: 1001
+```
+
+### 3.2 Create Tables
+
+Let's go back to our Flink SQL cli and the first thing we need is some tables to work with.
+
+We will be using the [Kafka Flink SQL connector](https://nightlies.apache.org/flink/flink-docs-release-1.16/docs/connectors/table/kafka/) to read sensor information and sensor readings from the two Kafka topic we already created.
+The following block shows how to create a table from a Kafka topic.
+
+*Note:* We will be using the default catalogs as well as the default database.
+
+```shell
+CREATE TABLE readings (
+ sensorId STRING,
+ reading DOUBLE,
+ eventTime_ltz AS TO_TIMESTAMP_LTZ(`timestamp`, 3),
+ `ts` TIMESTAMP(3) METADATA FROM 'timestamp',
+ `timestamp` BIGINT,
+ WATERMARK FOR eventTime_ltz AS eventTime_ltz - INTERVAL '30' SECONDS
+) WITH (
+ 'connector' = 'kafka',
+ 'topic' = 'sensor.readings',
+ 'properties.bootstrap.servers' = 'kafka:29092',
+ 'properties.group.id' = 'group.sensor.readings',
+ 'format' = 'json',
+ 'scan.startup.mode' = 'earliest-offset',
+ 'json.timestamp-format.standard' = 'ISO-8601',
+ 'json.fail-on-missing-field' = 'false',
+ 'json.ignore-parse-errors' = 'true'
+);
+
+
+ Flink SQL> DESCRIBE readings;
++---------------+----------------------------+------+-----+-------------------------------------+----------------------------------------+
+| name | type | null | key | extras | watermark |
++---------------+----------------------------+------+-----+-------------------------------------+----------------------------------------+
+| sensorId | STRING | TRUE | | | |
+| reading | DOUBLE | TRUE | | | |
+| eventTime_ltz | TIMESTAMP_LTZ(3) *ROWTIME* | TRUE | | AS TO_TIMESTAMP_LTZ(`timestamp`, 3) | `eventTime_ltz` - INTERVAL '30' SECOND |
+| ts | TIMESTAMP(3) | TRUE | | METADATA FROM 'timestamp' | |
+| timestamp | BIGINT | TRUE | | | |
++---------------+----------------------------+------+-----+-------------------------------------+----------------------------------------+
+5 rows in set
+```
+
+The *CREATE TABLE* syntax consists of column definitions, watermarks and connector properties (more details [here](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sql/create/#create-table)).
+
+We can observe the following column types in Flink SQL:
+1. *Physical (or regular) columns*
+2. *Metadata columns:* like the `ts` column in our statement that is basically Kafka metadata for accessing the `timestamp` from a Kafka Record.
+3. *Computed columns:* virtual columns like the *eventTime_ltz* in our statement, that is a formatted timestamp derived from our `timestamp` BIGINT column. Virtual Columns can reference other columns, perform simple computations (like 5 * reading) or use built-in functions
+
+*Note:* Specifying a Time attributes (here _eventTime_ltz_) and watermarks is what allows us to operate properly on even time and also set constraints on temporal operators, which we will shortly.
+
+Let's also create a table for our sensor information topic.
+```shell
+CREATE TABLE sensors (
+ sensorId STRING,
+ latitude String,
+ longitude String,
+ sensorType STRING,
+ generation STRING,
+ deployed BIGINT
+) WITH (
+ 'connector' = 'kafka',
+ 'topic' = 'sensor.info',
+ 'properties.bootstrap.servers' = 'kafka:29092',
+ 'properties.group.id' = 'group.sensor.info',
+ 'format' = 'json',
+ 'scan.startup.mode' = 'earliest-offset',
+ 'json.timestamp-format.standard' = 'ISO-8601',
+ 'json.fail-on-missing-field' = 'false',
+ 'json.ignore-parse-errors' = 'true'
+);
+
+Flink SQL> DESCRIBE sensors;
++------------+--------+------+-----+--------+-----------+
+| name | type | null | key | extras | watermark |
++------------+--------+------+-----+--------+-----------+
+| sensorId | STRING | TRUE | | | |
+| latitude | STRING | TRUE | | | |
+| longitude | STRING | TRUE | | | |
+| sensorType | STRING | TRUE | | | |
+| generation | STRING | TRUE | | | |
+| deployed | BIGINT | TRUE | | | |
++------------+--------+------+-----+--------+-----------+
+```
+You might wonder why there is no time attribute or watermark specified on this topic.
+
+The sensor information is basically used to hold *state* - i.e it's a *changelog stream* backed by a compacted Kafka topic (compared to an *append-only stream*) because we are only interested in keeping the latest sensor
+information to perform event enrichment later (when we will discuss joins.)
+
+At this point we have two tables created within our default database.
+```shell
+Flink SQL> SHOW TABLES;
+>
++------------+
+| table name |
++------------+
+| readings |
+| sensors |
++------------+
+2 rows in set
+```
+
+Next up we need some data in our topics (and tables) to work with.
+I will use the following Producer code that can be found [here](https://github.com/polyzos/select-star-from-streams/blob/main/src/main/kotlin/io/ipolyzos/producers/SensorProducer.kt)
+to generate information for _10 sensors_ and _10.000 readings_. You can modify the code to ingest more data if you want.
+
+*Note:* I skipped added the implementation here since our focus is on Flink SQL.
+
+### 3.3 Run our First Query
+Let's run the following query and see an output similar to the following with information for our 10 sensors.
+
+```shell
+SELECT * FROM sensors;
+
+------- Sample Output -------
+
++----+--------------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+----------------------+
+| op | sensorId | latitude | longitude | sensorType | generation | deployed |
++----+--------------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+----------------------+
+| +I | 1 | 83.964156 | 47.567865 | PROXIMITY | 0 | 1610920880369 |
+| +I | 2 | 70.211600 | 87.285699 | PROXIMITY | 0 | 1669725385766 |
+| +I | 3 | -78.843922 | -159.70556 | TEMPERATURE | 2 | 1645179854537 |
+| +I | 4 | 77.304485 | 102.32052 | PROXIMITY | 2 | 1596841078647 |
+| +I | 5 | -11.876575 | 57.576944 | PROXIMITY | 3 | 1667359403631 |
+| +I | 6 | 59.134005 | -159.71549 | PROXIMITY | 0 | 1604501300175 |
+| +I | 7 | -16.478654 | 141.49999 | TEMPERATURE | 1 | 1614461145113 |
+| +I | 8 | -64.380075 | 164.37186 | PROXIMITY | 2 | 1673640554153 |
+| +I | 9 | -33.693995 | -2.4277239 | TEMPERATURE | 3 | 1645551899832 |
+| +I | 10 | -88.115880 | 11.500759 | PROXIMITY | 2 | 1623336384463 |
+```
+
+## 4. Operators
+
+
+
+### 4.1 Stateless Operators
+Stateless Operators are the simplest and include common operations like Projections and Filters that require no state.
+
+*Query* Only Sensor Readings > 40
+```shell
+SELECT sensorId, reading, eventTime_ltz
+FROM readings
+WHERE reading > 40
+
+------- Sample Output -------
+
++----+--------------------------------+--------------------------------+-------------------------+
+| op | sensorId | reading | eventTime_ltz |
++----+--------------------------------+--------------------------------+-------------------------+
+| +I | 1 | 40.18 | 2023-01-30 20:17:45.297 |
+| +I | 1 | 41.87 | 2023-01-30 20:17:45.334 |
+| +I | 1 | 41.72 | 2023-01-30 20:17:45.577 |
+| +I | 8 | 40.91 | 2023-01-30 20:17:45.825 |
+| +I | 5 | 40.94 | 2023-01-30 20:17:46.030 |
+| +I | 7 | 40.73 | 2023-01-30 20:17:46.164 |
+| +I | 5 | 40.13 | 2023-01-30 20:17:46.468 |
+| +I | 5 | 40.22 | 2023-01-30 20:17:46.495 |
+| +I | 7 | 40.02 | 2023-01-30 20:17:46.890 |
+| +I | 7 | 40.92 | 2023-01-30 20:17:46.971 |
+```
+
+### 4.2 Materializing Operators
+Materializing Operators perform computations that are not constrained by temporal conditions and thus never complete - the input / output records are constantly updated or deleted.
+
+Consider a *GROUP BY sensorId operation*. The query needs to maintain *state* for every sensorId, in order to update the results accordingly each time a new event for a sensor arrives.
+
+This means the state is kept around forever and constantly growing with every new sensor generated event.
+
+*Query:* Total readings per Sensor
+```shell
+SELECT sensorId, COUNT(reading) as totalReadings
+FROM readings
+GROUP BY sensorId
+
+------- Sample Output -------
+
++----+--------------------------------+----------------------+
+| op | sensorId | totalReadings |
++----+--------------------------------+----------------------+
+| +I | 4 | 1 |
+| -D | 4 | 1 |
+| +I | 6 | 1 |
+| -D | 6 | 1 |
+| +I | 4 | 2 |
+| -D | 4 | 2 |
+| +I | 6 | 2 |
+| -D | 6 | 2 |
+| +I | 4 | 6 |
+| -D | 4 | 6 |
+| +I | 6 | 3 |
+| -D | 6 | 3 |
+| +I | 4 | 7 |
+| -D | 4 | 7 |
+| +I | 6 | 7 |
+| -D | 6 | 7 |
+| +I | 4 | 8 |
+| -D | 4 | 8 |
+```
+*Notice the op column* - when we have an update for a given *key* the previous row is deleted and updated to the new value.
+For example
+```shell
+| +I | 4 | 1 |
+| -D | 4 | 1 |
+| +I | 4 | 2 |
+```
+
+Now consider a query that joins the *sensor information* and *sensor readings* tables.
+
+*Query:* Enrich Sensor readings with Sensor Information (Regular Join)
+```shell
+SELECT
+ sensors.sensorId,
+ reading,
+ eventTime_ltz,
+ latitude,
+ longitude,
+ sensorType
+FROM readings
+ JOIN sensors ON readings.sensorId = sensors.sensorId
+
+------- Sample Output -------
+
++----+--------------------------------+--------------------------------+-------------------------+--------------------------------+--------------------------------+--------------------------------+
+| op | sensorId | reading | eventTime_ltz | latitude | longitude | sensorType |
++----+--------------------------------+--------------------------------+-------------------------+--------------------------------+--------------------------------+--------------------------------+
+| +I | 1 | 40.18 | 2023-01-30 20:17:45.297 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 1 | 38.95 | 2023-01-30 20:17:45.301 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 1 | 41.87 | 2023-01-30 20:17:45.334 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 1 | 39.92 | 2023-01-30 20:17:45.375 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 1 | 39.28 | 2023-01-30 20:17:45.408 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 7 | 39.99 | 2023-01-30 20:17:45.443 | -16.478654 | 141.49999 | TEMPERATURE |
+| +I | 1 | 38.27 | 2023-01-30 20:17:45.551 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 7 | 38.46 | 2023-01-30 20:17:45.553 | -16.478654 | 141.49999 | TEMPERATURE |
+| +I | 1 | 41.72 | 2023-01-30 20:17:45.577 | 83.964156 | 47.567865 | PROXIMITY |
+| +I | 7 | 40.73 | 2023-01-30 20:17:46.164 | -16.478654 | 141.49999 | TEMPERATURE |
+
+```
+Both tables are kept in memory which means the state will keep growing for both sides of the joins and thus it's important to
+expire state by using a ttl.
+
+You can achieve this using [`table.exec.state.ttl`](https://nightlies.apache.org/flink/flink-docs-release-1.16/docs/dev/table/config/#table-exec-state-ttl).
+Keep in mind though that there is a tradeoff between accuracy and state size, as expiring state too early might result in incomplete results.
+
+*If you need to keep really large state around you will need to configure Flink to use Rocksdb as a state backend,*
+
+Unlike Materializing Operators, Temporal Operators (that we will see next) will automatically expire state that is no longer useful, by using time constraints - although you may need to set an idle state retention interval
+
+For example a time Window that is considered _complete_ doesn't need to be kept around in state.
+
+### 4.3 Temporal Operators
+Temporal Operators are constraint by time. Records and computations are associated with a temporal condition, i.e a time window of 30 seconds
+and accept new records - previously added records can not be updated or deleted.
+
+As we previously mentioned they hold records and/or results in state, but only until they are no longer required.
+
+*Query:* Find the 1 minute average value for each sensor
+```shell
+SELECT
+ sensorId,
+ window_start,
+ window_end,
+ COUNT(reading) AS totalReadings,
+ LISTAGG(CAST(reading AS STRING)) AS readingsList,
+ ROUND(AVG(reading),1) as averageReading
+FROM TABLE(TUMBLE(TABLE readings, DESCRIPTOR(eventTime_ltz), INTERVAL '1' MINUTE))
+GROUP BY sensorId, window_start, window_end
+
+------- Sample Output -------
+
++----+--------------------------------+-------------------------+-------------------------+----------------------+--------------------------------+--------------------------------+
+| op | sensorId | window_start | window_end | totalReadings | readingsList | averageReading |
++----+--------------------------------+-------------------------+-------------------------+----------------------+--------------------------------+--------------------------------+
+| +I | 4 | 2023-01-30 16:44:00.000 | 2023-01-30 16:45:00.000 | 17 | 40.59,40.17,39.98,39.66,40.... | 40.1 |
+| +I | 1 | 2023-01-30 16:44:00.000 | 2023-01-30 16:45:00.000 | 26 | 40.23,38.84,36.6,39.31,39.9... | 39.6 |
+| +I | 4 | 2023-01-30 16:45:00.000 | 2023-01-30 16:46:00.000 | 440 | 40.31,43.09,40.19,40.35,39.... | 39.9 |
+| +I | 1 | 2023-01-30 16:45:00.000 | 2023-01-30 16:46:00.000 | 469 | 41.03,40.12,40.7,38.88,40.8... | 40.0 |
+| +I | 1 | 2023-01-30 16:46:00.000 | 2023-01-30 16:47:00.000 | 469 | 39.49,39.42,40.09,40.66,38.... | 39.9 |
+| +I | 4 | 2023-01-30 16:46:00.000 | 2023-01-30 16:47:00.000 | 447 | 40.44,40.98,39.79,39.21,40.... | 40.0 |
+| +I | 4 | 2023-01-30 16:47:00.000 | 2023-01-30 16:48:00.000 | 459 | 36.82,40.19,39.66,39.83,42.... | 40.0 |
+| +I | 1 | 2023-01-30 16:47:00.000 | 2023-01-30 16:48:00.000 | 494 | 40.45,39.37,41.69,40.41,39.... | 40.1 |
+| +I | 1 | 2023-01-30 16:48:00.000 | 2023-01-30 16:49:00.000 | 494 | 40.35,39.02,41.26,37.56,41.... | 40.0 |
+| +I | 4 | 2023-01-30 16:48:00.000 | 2023-01-30 16:49:00.000 | 447 | 41.15,39.46,38.72,37.01,39.... | 40.0 |
+| +I | 2 | 2023-01-30 16:44:00.000 | 2023-01-30 16:45:00.000 | 20 | 39.12,41.12,41.68,38.75,39.... | 40.3 |
+| +I | 2 | 2023-01-30 16:45:00.000 | 2023-01-30 16:46:00.000 | 497 | 40.07,39.5,38.64,39.95,40.5... | 40.0 |
+```
+
+*Query:* Find reading statistics (max, min, average and stddev) for all readings per sensorId over the previous minute.
+```shell
+SELECT
+ eventTime_ltz,
+ sensorId,
+ reading,
+ ROUND(AVG(reading) OVER minuteInterval, 1) AS minuteAvgTemp,
+ MAX(reading) OVER minuteInterval AS minuteMinTemp,
+ MIN(reading) OVER minuteInterval AS minuteMaxTemp,
+ ROUND(STDDEV(reading) OVER minuteInterval, 5) AS minuteStdevTemp
+FROM readings
+WINDOW minuteInterval AS (
+ PARTITION BY sensorId
+ ORDER BY eventTime_ltz
+ RANGE BETWEEN INTERVAL '1' MINUTE PRECEDING AND CURRENT ROW
+);
+```
+
+### 5. (Temporary) Views
+As we mentioned Flink SQL is quite rich and provides a lot of function - so covering everything in this article is impossible.
+One more useful feature though I want to mention is *Temporary Views*.
+
+Similar to database Views it can be used to store the results of a query.
+A view is not physically materialized, but instead it is run every time the view is referenced in a query.
+Temporary Views are very useful to structure and decompose more complicated queries or reuse queries within other queries.
+
+Once more let us better illustrate this with an example.
+
+Following our previous query that calculates statistics we can make use of Temporary Views to store the output
+of the query and reuse it the calculated statistics to filter readings; for example find *readings > average + 2 * standard deviation*.
+
+That's a simple example that can be used to build more sophisticated outlier detection logic.
+
+```shell
+--- Create a Temporary View -- CREATE [TEMPORARY] VIEW
+CREATE VIEW readings_stats AS
+SELECT
+ eventTime_ltz,
+ sensorId,
+ reading,
+ ROUND(AVG(reading) OVER minuteInterval, 1) AS minuteAvgTemp,
+ MAX(reading) OVER minuteInterval AS minuteMinTemp,
+ MIN(reading) OVER minuteInterval AS minuteMaxTemp,
+ ROUND(STDDEV(reading) OVER minuteInterval, 5) AS minuteStdevTemp
+FROM readings
+WINDOW minuteInterval AS (
+ PARTITION BY sensorId
+ ORDER BY eventTime_ltz
+ RANGE BETWEEN INTERVAL '1' MINUTE PRECEDING AND CURRENT ROW
+);
+
+--- Run a filter query on the results to get the readings we want
+SELECT
+ sensorId,
+ reading,
+ ROUND(minuteAvgTemp + 2 * minuteStdevTemp, 2) as threshold
+FROM readings_stats
+WHERE reading > minuteAvgTemp + 2 * minuteStdevTemp
+
+------- Sample Output -------
++----+--------------------------------+--------------------------------+--------------------------------+
+| op | sensorId | reading | threshold |
++----+--------------------------------+--------------------------------+--------------------------------+
+| +I | 5 | 41.6 | 41.42 |
+| +I | 3 | 42.1 | 41.69 |
+| +I | 5 | 41.6 | 41.52 |
+| +I | 3 | 42.6 | 42.52 |
+| +I | 5 | 41.7 | 41.69 |
+| +I | 3 | 41.5 | 41.38 |
+| +I | 7 | 41.1 | 41.07 |
+| +I | 4 | 42.5 | 41.97 |
+| +I | 1 | 41.3 | 41.2 |
+| +I | 1 | 41.8 | 41.74 |
+```
+
+## 6. The TableEnvironment and SQL Queries
+Up to this point we have been using the Flink SQL cli to submit sql queries.
+For production cases though - or if you are running in environments like Kubernetes for example using the [Flink Operator](https://github.com/apache/flink-kubernetes-operator), you might need other ways to achieve this.
+
+*Note 1:* Flink *1.16* introduced [*Flink SQL Gateway*](https://nightlies.apache.org/flink/flink-docs-release-1.16/docs/dev/table/sql-gateway/overview/) that you can use to submit queries.
+
+Next we will see how we can use the TableEnvironment to run such queries through code.
+
+*Note 2:* If you are running on kubernetes using the Flink Operator you might wanna also check these example [here](https://github.com/apache/flink-kubernetes-operator/tree/main/examples/flink-sql-runner-example)
+
+*Note 3:* Seeing the sample code below might seem weird as I'm using Kotlin. Whether you are using Java, Kotlin or Scala should be exactly the same - I'm just using kotlin these days and because Java 17 is unfortunately not supported yet at Flink I wanted to leverage Kotlin for multiline strings to write my queries.
+
+### 6.1 Running SQL Queries with Code
+```kotlin
+class EnrichmentStream {
+ private val checkpointsDir = "file://${System.getProperty("user.dir")}/checkpoints/"
+ private val rocksDBStateDir = "file://${System.getProperty("user.dir")}/state/rocksdb/"
+
+ companion object {
+ @JvmStatic
+ fun main(args: Array) {
+ EnrichmentStream().runStream()
+ }
+ }
+
+ fun runStream() {
+ val environment = StreamExecutionEnvironment
+ .createLocalEnvironmentWithWebUI(Configuration())
+
+ environment.parallelism = 3
+
+ // Checkpoint Configurations
+ environment.enableCheckpointing(5000)
+ environment.checkpointConfig.minPauseBetweenCheckpoints = 100
+ environment.checkpointConfig.setCheckpointStorage(checkpointsDir)
+
+ val stateBackend = EmbeddedRocksDBStateBackend()
+ stateBackend.setDbStoragePath(rocksDBStateDir)
+ environment.stateBackend = stateBackend
+
+ environment.checkpointConfig.externalizedCheckpointCleanup =
+ CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION
+
+ // Configure Restart Strategy
+ environment.restartStrategy = RestartStrategies.fixedDelayRestart(5, Time.seconds(5))
+
+ val tableEnvironment = StreamTableEnvironment.create(environment)
+
+ // Run some SQL queries to check the existing Catalogs, Databases and Tables
+ tableEnvironment
+ .executeSql("SHOW CATALOGS")
+ .print()
+
+ tableEnvironment
+ .executeSql("SHOW DATABASES")
+ .print()
+
+ tableEnvironment
+ .executeSql("SHOW TABLES")
+ .print()
+
+ tableEnvironment
+ .executeSql(Queries.CREATE_SENSORS_TABLE)
+ .print()
+
+ tableEnvironment
+ .executeSql(Queries.CREATE_READINGS_TABLE)
+ .print()
+
+ tableEnvironment
+ .executeSql("SHOW TABLES")
+ .print()
+
+ tableEnvironment
+ .executeSql(Queries.JOIN_SENSOR_READINGS_WITH_INFO_QUERY)
+ .print()
+ }
+}
+```
+
+The TableEnvironment is the entrypoint for Table API and SQL integration and is responsible for:
+- Registering a Table in the internal catalog
+- Registering catalogs
+- Loading pluggable modules
+- Executing SQL queries
+- Registering a user-defined (scalar, table, or aggregation) function
+- Converting between DataStream and Table (in case of StreamTableEnvironment)
+
+The code snippet illustrated above runs the *Join Operations* we saw before (you can find the queries [here](https://github.com/polyzos/depths-of-flink/blob/main/src/main/kotlin/io/ipolyzos/compute/Queries.kt))
+and the output should be similar.
+
+### 6.2 A Short Discussion: Checkpoints and State
+As extras you might notice two things:
+1. Checkpoint is enabled
+2. Rocksdb as a state backend is enabled.
+
+We are not going into detail for these concepts here as it's a story for another day. For those interested if you run the code above you should see two output directories, `checkpoints` and `state`.
+
+
+
+Since we are consuming (at most) from 3 kafka partitions, I'm using a parallelism of 3 as you might have also noticed in the code and so the stateful operator that runs the join will
+store state for each of the three tasks.
+
+You can use a sample code file I have added [here](https://github.com/polyzos/depths-of-flink/blob/main/src/main/kotlin/io/ipolyzos/inspect/RocksDBInspect.kt) to see what gets written in state.
+```shell
+Processing state of operator: job_848920cbd5178c2a525827b244d1e530_op_StreamingJoinOperator_8b481b930a189b6b1762a9d95a61ada1__2_3__uuid_97fb2e4b-e6f4-413e-98a5-0c4ad24ed20e}
+ Column Family 'right-records' has 3 entries.
+ Column Family 'left-records' has 670 entries.
+Processing state of operator: job_848920cbd5178c2a525827b244d1e530_op_StreamingJoinOperator_8b481b930a189b6b1762a9d95a61ada1__3_3__uuid_46df8b7d-d015-4cc3-ad53-b86b7ecb0f1e}
+ Column Family 'right-records' has 4 entries.
+ Column Family 'left-records' has 879 entries.
+Processing state of operator: job_848920cbd5178c2a525827b244d1e530_op_StreamingJoinOperator_8b481b930a189b6b1762a9d95a61ada1__1_3__uuid_6166550d-6e03-489f-94ab-4004ea7ec50e}
+ Column Family 'right-records' has 3 entries.
+ Column Family 'left-records' has 521 entries.
+```
+You can see that we have two [column families](https://github.com/EighteenZi/rocksdb_wiki/blob/master/Column-Families.md) - one for the left side of the join (sensor readings) and one for the right (sensor information).
+Notice for example for the `right-records` since we have _10 sensor ids_ these keys are distributed among the three tasks.
+
+Also note as discussed previously since there are no time constraints you will notice the state growing indefinitely (unless `table.exec.state.ttl` is configured).
+
+I hope I sparked some interest for those curious that want to dive deeper themselves.
+In this article though we will conclude with this high level overview.
+
+## 7. Wrap Up
+Flink is a powerful Stateful Stream Processing engine, enabling Unified Batch and Streaming architectures.
+
+Flink SQL is a high level API, using the well-known SQL syntax making it easy for everyone - like scientists or non-JVM (or python) engineers to leverage the power of Stream Processing with Apache Flink.
+
+Flink SQL is extremely rich and supports a wide-variety of built-in operators and functions
+
+Unless there are some really sophisticated use cases (that can not be expressed in SQL and need low-level Datastream API access) Flink SQL is the best candidate for Stream Processing
diff --git a/_posts/2023-02-13-akkastreams-to-actors-and-back.md b/_posts/2023-02-13-akkastreams-to-actors-and-back.md
new file mode 100644
index 000000000000..88eb41a91788
--- /dev/null
+++ b/_posts/2023-02-13-akkastreams-to-actors-and-back.md
@@ -0,0 +1,534 @@
+---
+title: "From Akka Streams To Actors And Back"
+date: 2023-02-13
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [akka]
+excerpt: "Interoperability between Akka Streams and actors with code examples"
+---
+
+_This article is brought to you by [Niklas Uhrberg](https://github.com/niklasuhrberg), a Rock the JVM alumnus who is a software architect with a focus on distributed systems. He's passionate about Scala, Akka and Kafka, and this piece is derived from his real-world experience._
+
+> If you want to learn Akka, we have more than 40 hours of training here at Rock the JVM in the [Akka Bundle](https://rockthejvm.com/p/the-akka-bundle), which ties in very well with this article.
+
+_Enter Niklas:_
+
+Akka Streams and Akka actors-based code have their distinct properties and bridging between these two programming models
+is useful in some scenarios. This is an account of a way to enhance the standard ways to bridge the gap. To make the
+most out of this text the reader should know Akka Streams ([taught here](https://rockthejvm.com/p/akka-streams) at Rock the JVM) and actors and ideally have some experience using them together. The reader is encouraged to study the example code on [GitHub](https://github.com/niklasuhrberg/akka-streamtoactor-interop) (see references) to get the full context and to experiment with the examples. In real world scenarios, using stream elements that originate from actors or enter an actor
+based part of a system is an excellent way of connecting Akka Streams to other types of processing.
+
+## 1. Motivation - Stream to Actor Scenario
+
+Let's consider a common scenario: subscribing to a Kafka topic and processing the messages. Assume that the processing
+of each message involves actor based processing, i.e. not just processing in Akka flows within an Akka stream. Depending on your requirements, this may not be so simple.
+
+There are two standard approaches, both of which are described in the Akka documentation:
+
+1. Define a Sink using `ActorSink.actorRefWithBackpressure` with a protocol for backpressure and lifecycle, i.e. messages
+ to initiate, finish and ack to control backpressure. Then connect the sink to the source, where Alpakka Kafka is often
+ used.
+
+ 
+
+2. Use the ask pattern in an Akka Flow. This can be done in plain vanilla style or with the `ActorFlow.ask` flow which is
+ just a small convenience utility.
+
+ 
+
+Now to the core of why this article was written: Neither of these two approaches or variants thereof satisfy all the
+following reasonable requirements:
+
+1. The Kafka commit must not happen until the processing in "actor land" is completed.
+2. There must be no arbitrariness about the error handling, i.e. no unknown states or outcomes.
+3. An error in the stream part must be propagated to the actor part.
+4. The actor based part must be able to process messages asymmetrically, i.e. sometimes receive more or fewer messages
+ than emitted downstream.
+
+## 2. Analysis of the Standard Approaches
+
+The actor sink based approach has no room for committing to Kafka conditionally, if you don't go at length and commit
+inside the actor part in some way. In short, if you do not use auto commit, it's an impractical setup. You will
+recognize this approach sketched in
+
+```scala
+val source = akka.kafka.scaladsl.Consumer
+ .committableSource(...) // In practice autocommit will be enabled
+val sink = Sink.actorRefWithBackpressure(
+ targetActor, // The actor processing the logic
+ onInitMessage = InitMessage, // Backpressure and lifecycle protocol
+ ackMessage = AckMessage,
+ onCompleteMessage = OnCompleteMessage,
+ onFailureMessage = onErrorMessage)
+
+source.runWith(sink)
+```
+
+I will leave it at that and instead discuss the ask based approach more in detail since it has a straight forward
+solution for the Kafka commits.
+
+Let's start by going through the requirements above. Akka Streams has excellent support for committing to Kafka in
+different ways. See the code snippet where an Akka source is connected and run with the Alpakka committer sink.
+
+This way the messages will not be committed in case the stream fails before reaching the committer sink and the commit
+step can be skipped based on application logic by simply not propagating the message all the way to the committer sink
+(see the example application on GitHub accompanying this article).
+
+The snippet below intentionally leaves out an interesting part in the SomePassthroughFlow. The role of this flow is to
+propagate the Kafka related information needed by the sink, but being irrelevant for the actor. If we do not use a
+pass-through flow we would have to pollute the actor message protocol with Kafka related information which is clearly
+not desirable. (Describing a pass-through flow is outside the scope of this article, it is a standard concept in
+streaming applications)
+
+```scala
+val source = akka.kafka.scaladsl.Consumer
+ .committableSource(...)
+val actorFlow = ActorFlow.ask[Int, Invocation, ProcessMessage](targetActor) { (in, ref) =>
+ Invocation(s"Invocation no $in", ref)
+}
+
+sampleSource
+ .via(SomePassThroughFlow(actorflow))
+ .runWith(Committer.sink(CommitterSettings(system))
+```
+
+The remaining requirements 2–4 are more problematic.
+
+The ask pattern is used with a timeout that, while suitable for many scenarios, sometimes may not make sense. A
+timeout leaves the stream part clueless about what happened in the actor processing part, all it knows is that it has
+taken longer time that it was willing to wait. If the ask times out everything in the actor part may have gone perfectly
+well, only that it took longer time than expected. You might therefore be inclined not to use an approach that mandates
+a timeout.
+
+Next to requirement 3 about the actor being notified if the stream fails which is clearly a cumbersome shortcoming of
+the ask based approach should you need this. Study the flow below for a small snippet from the accompanying source code
+for this article.
+
+```scala
+val askFlow: Flow[Int, ProcessMessage, NotUsed] =
+ ActorFlow.ask[Int, Invocation, ProcessMessage](targetActor)((in, q) => Invocation(s"Invocation no $in", q))
+
+Source(1 to 5)
+ .map {
+ case 3 =>
+ logger.debug(s"Step 1, 3 arrived, will throw exception")
+ throw new IllegalStateException("3 is a bad number")
+ case other => other
+ }
+ .via(askFlow)
+ .runWith(Sink.foreach(response => logger.debug(s"Got response $response"))
+```
+
+The target actor (not defined in the snippet) will never know if the stream fails or completes unless this is handled
+separately and explicitly. You therefore face risks, for example that of not cleaning up resources. It should be noted
+though that it is possible to e.g. handle the notification of the target actor outside the stream itself, by
+connecting that functionality to the completion or failure of the stream. See the following snippet for an example.
+
+```scala
+val futureDone = Source(1 to 5)
+ .via(askFlow)
+ .runWith(Sink.foreach(response => logger.debug(s"Got response $response")))
+ // This handles the propagation of stream completion or failure to the actor
+ futureDone.onComplete {
+ case Success(value) => logger.debug("Stream with ask completed successfully")
+ targetActor ! StreamCompleted
+ case Failure(exception) => logger.error(s"Stream with ask failed with ${exception.getMessage}")
+ targetActor ! StreamFailed
+ }
+```
+
+Finally, to requirement 4 which is about the multiplicity of messages flowing through the actor based part. In the ask
+based approach there is only one possibility, one message in and one message out (the ask response). If you need to send
+multiple responses this is simply not possible, but in Akka Streams there is nothing preventing you from emitting
+multiple messages downstream from a flow no matter what happens upstream and vice versa, having a flow consuming
+messages without emitting. This is a perfectly viable usage in the example use case with Kafka message processing.
+
+Let's assume that some messages cannot be processed immediately, and you wish to store them for later processing, for example
+using Akka Persistence, but still continue fetching more Kafka messages. Then it makes perfect sense to sometimes emit
+more messages from the actor based flow to enable monotonically increasing offset for the commit step. You might have
+some messages that should have a commit as a consequence and others that do not, if each Kafka message is firstly
+processed by either storing it or handling the message immediately and then the stored messages are processed later
+causing emit downstream but without Kafka commit.
+
+```scala
+def inflowFromKafka(messageIn:KafkaMessageWithCommitInfo)= {
+ if(shouldStoreForLater(messageIn)) {
+ store(messageIn)
+ emitMessageSignallingKafkaCommit
+ }
+}
+
+def processStoredMessage = {
+ val message = obtainASuitablStoredMessageForProcessing
+ emitMessageSignallingProcessedResult(process(message))
+}
+```
+
+The pseudocode above expresses some aspects of the use case sketched to show that there can be multiple emitted
+messages for one message in. One Kafka message in can result in one message out to signal Kafka commit because the
+information has been successfully stored and then later when the stored information is properly processed achieving the
+desired result, it is propagated downstream using a second message. This is not possible to achieve with the ask based
+approach, leading us to the next approach.
+
+## 3. The StageActor Approach
+Now that we can appreciate the pros and cons using the standard approaches with an actor based sink and the variants of
+actor based flows it's time to present an alternative approach that will satisfy all the requirements described above.
+The general setup has the same anatomy as the ask based flow with the important difference that the communication with
+the target actor is now based on actor messaging which is not restricted to request - response style.
+
+
+
+ActorRefBackpressureProcessFlowStage is the name of a custom GraphStage used in the example code and
+StreamToActorMessage[T] is the messaging protocol used to control message passing and backpressure.
+To set up a stream according to the diagram above we will have the target actor process StreamToActorMessage[T] messages
+and construct the ActorRefBackpressureProcessFlowStage passing the actor ref of the target actor.
+The StreamToActorMessage[T] message protocol will be described in detail, for now we note that they serve the same
+purpose as in the sink based approach to manage the stream semantics.
+
+```scala
+val targetActor = ctx.actorOf(PropsAdapter[StreamToActorMessage[FlowMessage]](TargetActor()))
+val actorFlow: Flow[FlowMessage, FlowMessage, NotUsed] =
+ Flow.fromGraph(new ActorRefBackpressureProcessFlowStage[FlowMessage](targetActor))
+```
+
+Internally the ActorRefBackpressureProcessFlowStage uses the stage actor, a perhaps not so well known part of the
+Akka Streams toolbox. In short, the stage actor (although it's not an actor per se) imitates some actor functionality
+and can be used inside a custom GraphStage to communicate with the "outside world".
+
+The following is an excerpt from the GraphStage documentation:
+
+```scala
+
+/*
+Initialize a StageActorRef which can be used to interact with from the outside world "as-if" an [[Actor]].
+The messages are looped through the getAsyncCallback mechanism of GraphStage so they are safe to modify
+internal state of this operator.
+ */
+
+final protected def getStageActor(receive: ((ActorRef, Any)) => Unit): StageActor
+
+/**
+ * Minimal actor to work with other actors and watch them in a synchronous ways
+ *
+ * Not for user instantiation, use [[#getStageActor]].
+ */
+final class StageActor @InternalApi() private[akka]
+```
+
+By creating a custom Flow component using the stage actor we can create a bridge from the stream to the actor based part
+of an application that has all the properties we desire. Let's now see in some more detail exactly how this is achieved.
+We start out with a custom GraphStage, `ActorRefBackpressureProcessFlowStage`, from which we want to communicate with
+the actor based part of an application. The point of communication with this is naturally an actor, the target actor
+embodying "the outside world" mentioned in the Akka docs above. The communication from ActorRefBackpressureProcessFlowStage
+to the target actor is just ordinary actor message passing.
+The first message passed to the target actor in this example:
+
+```scala
+// In ActorRefBackpressureProcessFlowStage
+targetActor ! StreamInit[T](self.ref) //self.ref is the stage actor ref
+```
+
+and in the target actor
+
+```scala
+case StreamInit(replyTo) => // replyTo is the stage actor
+ logger.debug("Received StreamInit, will signal demand by responding ack")
+ replyTo ! StreamAck
+ Behaviors.same
+```
+
+and the other way is mediated via the stage actor. An abbreviated example:
+
+```scala
+getStageActor({
+ case (_, StreamAck) =>
+ onAck()
+ case StreamElementIn(msg:Invocation, replyTo) =>
+ case (_, StreamElementOut(elemOut)) =>
+ onElementOut(elemOut)
+// Only some cases mentioned here for brevity
+})
+```
+
+The messaging protocol used between the stage actor and the target actor serves two purposes. Communicating the
+application specific information and the stream lifecycle and backpressure signalling. `StreamToActorMessage[T]` is just
+the trait used in the example code, where T is the type of the application specific message that is tunneled to the
+target actor.
+The `StreamToActorMessage` trait contains virtually the same type of messages used in the sink based approach.
+
+```scala
+case class StreamInit[T](replyTo:ActorRef[StreamToActorMessage[T]]) extends StreamToActorMessage[T]
+case object StreamAck extends StreamToActorMessage[Nothing]
+case object StreamCompleted extends StreamToActorMessage[Nothing]
+case class StreamFailed(ex: Throwable) extends StreamToActorMessage[Nothing]
+case class StreamElementIn[T](msg: T, replyTo:ActorRef[StreamToActorMessage[T]]) extends StreamToActorMessage[T]
+case class StreamElementOut[T](msg: T) extends StreamToActorMessage[T]
+case class StreamElementOutWithAck[T](msg:T) extends StreamToActorMessage[T]
+```
+
+The interaction starts with the target actor receiving a StreamInit message. This is when it gets hold of the stage
+actor ref and can start communicating with it. The replyTo in this message is a reference to the stage actor.
+At some point in time the target actor responds with a StreamAck message to signal demand, which triggers the stream
+to start processing. The next step typically is the StreamElementIn message through which the application specific
+message (of type T) is tunneled. At this point the target actor may either signal more demand, cause downstream emit
+by responding StreamElementOut or both in any order it sees fit.
+
+StreamCompleted will be sent to the target actor on stream completion, and StreamFailed on stream failure.
+The target actor may use StreamElementOutWithAck as a convenience combining emitting an element and signalling demand at the same time. Communicating failure from the target actor to the stream is triggered by the target actor stopping which is detected by the stage actor watching the target actor. This is a full-bodied protocol for managing stream processing both ways including completion and failure.
+
+In the vocabulary of Akka Streams the ActorRefBackpressureProcessFlowStage:
+
+* __Emits__ when the target actor sends a message to the stage actor and downstream signals demand
+* __Backpressures__ unless the target actor signals demand
+* __Completes when__ the upstream completes
+* __Fails__ when the target actor terminates
+* __Cancels__ when downstream cancels
+
+The stage based actor approach to this problem has been addressed e.g. by Frank van Meeuwen. I have used the code with
+kind permission in this article and the original source code can be found on GitHub. (See references)
+
+## 4. Putting the StageActor to Use
+
+It's about time to take a look at the example code with some code snippets.
+Example target actor code showing two elements out for each element in:
+
+```scala
+def apply():Behavior[StreamToActorMessage] = Behaviors.receiveMessage[StreamToActorMessage] {
+ case StreamInit(replyTo) =>
+ replyTo ! StreamAck // The first interaction
+ Behaviors.same
+
+ case StreamElementIn(msg, replyTo) =>
+ replyTo ! StreamElementOut(Response(msg.name, Random.nextInt(100)))
+ replyTo ! StreamElementOut(Response(msg.name, Random.nextInt(100))
+ replyTo ! StreamAck // Signal more demand
+ Behaviors.same
+}
+```
+
+Remember that the stage actor approach completely decouples the rate of elements consumed by the custom flow from
+the rate of elements emitted by it.
+
+A simple example putting the central piece in a context that is runnable:
+
+```scala
+object StreamWithStageActor extends LazyLogging {
+
+ implicit val timeout = Timeout(5.seconds)
+
+ def apply(): Behavior[Nothing] = Behaviors.setup[Nothing] { ctx =>
+ implicit val system: ActorSystem[_] = ctx.system
+ import system.executionContext
+ // The PropsAdapter is necessary because the stage actor is classic Akka
+ val targetActor = ctx.actorOf(PropsAdapter[StreamToActorMessage[FlowMessage]](TargetActor()))
+ val actorFlow: Flow[FlowMessage, FlowMessage, NotUsed] =
+ Flow.fromGraph(new ActorRefBackpressureProcessFlowStage[FlowMessage](targetActor))
+ val value = Source(1 to 3)
+ .map(number => Invocation(s"Name $number"))
+ .via(actorFlow)
+ .log("logflow", response => s"Got response $response")
+ .runWith(Sink.ignore)
+
+ value onComplete {
+ case Success(value) => logger.debug("Stream completed successfully")
+ case Failure(exception) => logger.error(s"Stream failed with $exception")
+ }
+ Behaviors.empty
+ }
+}
+```
+
+In the accompanying code on GitHub, there are more involved streams examples both using the ask based approach and
+the stage actor based approach to make it easy to play around with different variants that show how recovering,
+throwing exceptions both in the stream part and in the actor part will affect the behavior.
+See the comments in the code and README file.
+
+## 5. Inside the custom GraphStage
+
+Lastly a few glimpses of the ActorRefBackpressureProcessFlowStage code. While explaining the GraphStage is outside
+the scope of this article, just peeking into a couple of part is enlightening.
+
+Basically a custom GraphStage is created by encoding the behavior when elements flow in, emit elements and keep track
+of the logic for when demand should be signaled and the lifecycle functionality.
+
+At the outset the preStart method will be called, which is pretty self-explanatory. Self, the actual stage actor,
+watches our target actor to manage the failure logic. Also, the StreamInit message is passed onto the target actor to
+bootstrap the interactions.
+
+```scala
+private lazy val self = getStageActor(stageActorReceive)
+
+override def preStart(): Unit = {
+ self.watch(targetActor)
+ tellFlowActor(StreamInit[T](self.ref))
+ expectingAck = true
+}
+```
+
+The `stageActorReceive` method defines the stage actor behavior. Without going into details the code should give a feeling of the
+translational functionality.
+
+```scala
+def stageActorReceive(messageWithSender: (ActorRef, Any)): Unit = {
+ def onAck(): Unit = {
+ firstAckReceived = true
+ expectingAck = false
+ pullIfNeeded()
+ completeStageIfNeeded()
+ }
+
+ def onElementOut(elemOut: Any): Unit = {
+ val elem = elemOut.asInstanceOf[T]
+ emit(out, elem)
+ }
+
+ messageWithSender match {
+ case (_, StreamAck) =>
+ onAck()
+ case (_, StreamElementOut(elemOut)) =>
+ onElementOut(elemOut)
+ case (_, StreamElementOutWithAck(elemOut)) =>
+ onElementOut(elemOut)
+ onAck()
+ case (actorRef, Failure(cause)) =>
+ terminateActorAndFailStage(new RuntimeException(s"Exception during processing by actor $actorRef: ${cause.getMessage}", cause))
+
+ case (_, Terminated(targetRef)) =>
+ failStage(new WatchedActorTerminatedException("ActorRefBackpressureFlowStage", targetRef))
+
+ case (actorRef, unexpected) =>
+ terminateActorAndFailStage(new IllegalStateException(s"Unexpected message: `$unexpected` received from actor `$actorRef`."))
+ }
+}
+```
+
+On the other side, in the GraphStage handlers api we can see the corresponding for the stream interaction.
+The InHandler and OutHandler traits are the core of the api.
+
+```scala
+setHandler(in, new InHandler {
+ override def onPush(): Unit = {
+ val elementIn = grab(in)
+ targetActor ! StreamElementIn[T](elementIn, self.ref)
+ expectingAck = true
+ }
+
+ override def onUpstreamFailure(ex: Throwable): Unit = {
+ self.unwatch(targetActor)
+ targetActor ! StreamFailed(ex)
+ super.onUpstreamFailure(ex)
+ }
+
+ override def onUpstreamFinish(): Unit = {
+ if(!expectingAck) {
+ self.unwatch(targetActor)
+ targetActor ! StreamCompleted)
+ super.onUpstreamFinish()
+ }
+ }
+})
+
+setHandler(out, new OutHandler {
+ override def onPull(): Unit = {
+ if (!firstPullReceived) {
+ firstPullReceived = true
+ pullIfNeeded() //Only do the first pull
+ }
+ }
+
+ override def onDownstreamFinish(cause: Throwable): Unit = {
+ self.unwatch(targetActor)
+ targetActor ! StreamCompleted)
+ super.onDownstreamFinish(cause)
+ }
+})
+```
+
+## 6. Another Take on Pass-Through
+
+The careful reader may have spotted a gap in the account of the stage based approach in the context of the original
+use case with Kafka where committing is a crucial part.
+
+In the ask based example we saw the pass-through flow in action, which basically zips together the element that should
+pass through (carrying the information later used for Kafka commits) with the response from the target actor.
+You might now ask what happens in the stage based approach if the target actor responds multiple times for a single
+message in. The answer is it will simply not work without some modification.
+
+The complexity of the solution depends on the case at hand, and one example solution is provided in the example code.
+There must be some management of state and a mapping from messages to the target actor and messages from the target actor.
+Sticking to the use case with Kafka commit information, the most common scenario will be that there is one message from
+the target actor that should be associated with the commit information and consequently the state can be deleted as soon
+as that message has been observed. There must be some identifier to map a response message to the original message to
+the target actor, and in an enterprise application setting this will already be present. The following is a
+brief explanation of the simple example provided in the accompanying code.
+
+The application messages must have an identifier:
+```scala
+object IdFlowMessages {
+ trait FlowMessageWithId {
+ def id: UUID
+ }
+ case class InvocationWithId(id: UUID, name: String) extends FlowMessageWithId
+ case class ResponseWithId(id: UUID, name: String, sum: Int) extends FlowMessageWithId
+}
+```
+
+The stream signature is now slightly different. The input to the stage actor based flow is a tuple of the application
+message and the pass-through and the output is a tuple of the application message and an option of the pass-through
+since not all flow elements will carry the pass-through on. Note that the term pass-through now has a slightly different
+meaning. It just means the pass-through will accompany one message following the `PassthroughActorRefBackpressureProcessFlowStage`.
+
+```scala
+val targetActor = ctx.actorOf(PropsAdapter[StreamToActorMessage[FlowMessageWithId]](PassthroughTargetActor()))
+val actorFlow: Flow[(FlowMessageWithId, PassThrough), (FlowMessageWithId, Option[PassThrough]), NotUsed] =
+ Flow.fromGraph(new PassthroughActorRefBackpressureProcessFlowStage[FlowMessageWithId, PassThrough](targetActor))
+```
+
+Finally, a peek into the modified custom GraphStage, `PassthroughActorRefBackpressureProcessFlowStage`. It makes the assumption
+about the application messages that they have an identifier and also keeps a mapping from this id to the pass-through internally.
+
+```scala
+class PassthroughActorRefBackpressureProcessFlowStage[T<:FlowMessageWithId, P]
+(private val targetActor: ActorRef) extends GraphStage[FlowShape[(T, P), (T, Option[P])]] {
+
+// ....
+
+val passThroughMessageMapping: mutable.Map[UUID, P] =
+ mutable.Map[UUID, P]()
+// ....
+
+override def onPush(): Unit = {
+ val elementIn = grab(in)
+ passThroughMessageMapping.put(elementIn._1.id, elementIn._2)
+ tellTargetActor(StreamElementIn[T](elementIn._1, self.ref))
+ expectingAck = true
+}
+
+def onElementOut(elemOut: Any): Unit = {
+ val elem = elemOut.asInstanceOf[T]
+ // .remove(elem.id) results in a Option that is folded over
+ passThroughMessageMapping
+ .remove(elem.id)
+ .fold(emit(out, (elem, None)))(value => emit(out, (elem, Some(value))))
+}
+```
+
+When an element arrives (onPush) its pass-through is associated with the identifier to be obtained later.
+When onElementOut is processed the stored pass-through element is removed once it is not needed anymore and this simple
+state management will be good enough for the normal case where not a very big number of elements are stored in the mapping
+keeping the necessary state.
+
+## 7. Conclusion
+
+We have studied a custom Akka flow component based of the stage actor that gives us all the functionality to create a
+full-blown interaction between Akka Streams and actor based code. This can be used in different use cases where the
+functionality needs to include parts interacting with actor based code, or other code all together where an actor is
+used as a bridge between the two. The standard approaches fall short of supporting different rate of input to output and
+do not integrate the failure logic as in the showcased code.
+
+### References
+
+1. Sample applications for the ask based and stage actor based approaches described in the article:
+ https://github.com/niklasuhrberg/akka-streamtoactor-interop
+2. Frank van Meeuwen GitHub repo: https://github.com/fmeeuw/akka-playground
+3. Akka Streams documentation on actor interoperability: https://doc.akka.io/docs/akka/current/stream/actor-interop.html
diff --git a/_posts/2023-03-08-ultimate-guide-to-java-virtual-threads.md b/_posts/2023-03-08-ultimate-guide-to-java-virtual-threads.md
new file mode 100644
index 000000000000..7ca6722e537e
--- /dev/null
+++ b/_posts/2023-03-08-ultimate-guide-to-java-virtual-threads.md
@@ -0,0 +1,1027 @@
+---
+title: "The Ultimate Guide to Java Virtual Threads"
+date: 2023-02-23
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "Project Loom and virtual threads promise to bring modern concurrency paradigms that we already found in Kotlin (coroutines) and Scala (Cats Effect and ZIO fibers) in Java. They are still in preview, but we can already enjoy them."
+toc: true
+toc_label: "In this article"
+---
+
+_Another tour de force by [Riccardo Cardin](https://github.com/rcardin). Riccardo is a proud alumnus of Rock the JVM, now a senior engineer working on critical systems written in Java, Scala and Kotlin._
+
+Version 19 of Java came at the end of 2022, bringing us a lot of exciting stuff. One of the coolest is the preview of some hot topics concerning Project Loom: _virtual threads_ ([JEP 425](https://openjdk.org/jeps/425)) and _structured concurrency_ ([JEP 428](https://openjdk.org/jeps/428)). Whereas still in a preview phase (to tell the truth, structured concurrency is still in the incubator module), the two JEPs promise to bring modern concurrency paradigms that we already found in Kotlin (coroutines) and Scala (Cats Effect and ZIO fibers) also in the mainstream language of the JVM: The Java programming language.
+
+Without further ado, let's first introduce virtual threads. As we said, both projects are still evolving, so the final version of the features might differ from what we will see here. Future articles to come will focus on structured concurrency and other cool features of Project Loom.
+
+## 1. Setup
+
+As we said, both the JEPs are still in the preview/incubation step, so we must enable them in our project. At the end of the article, we will give an example of a Maven configuration with all the needed dependencies and configurations. Here, we will just show the most important parts.
+
+First, we need to use a version of Java that is at least 19. Then, we must give the JVM the `--enable-preview` flag. Although we will not talk about structured concurrency, we set up the environment to access it. So, we need to enable and import the `jdk.incubator.concurrent` module. Under the folder `src/main/java`, we need to create a file named `module-info.java` with the following content:
+
+```java
+module virtual.threads.playground {
+ requires jdk.incubator.concurrent;
+}
+```
+
+The name of our module doesn't matter. We used `virtual.threads.playground`, but we can use any name we want. The important thing is that we need to use the `requires` directive to enable the incubator module.
+
+We'll use Slf4j to log something on the console. So, all the code snippets in this article will use the following logger:
+
+```java
+static final Logger logger = LoggerFactory.getLogger(App.class);
+```
+
+However, we won't use the `logger` object directly in our example but the following custom function `log`:
+
+```java
+static void log(String message) {
+ logger.info("{} | " + message, Thread.currentThread());
+}
+```
+
+In fact, the above function allows us to print some helpful information concerning virtual threads that will be very handy in understanding what's going on.
+
+Moreover, we'll also use Lombok to reduce the boilerplate code when dealing with checked exceptions. So, we'll use the `@SneakyThrows`, which lets us treat checked exceptions as unchecked ones (don't use it in production!). For example, we'll wrap the `Thread.sleep` method, which throws a checked `InterruptedException`, with the `@SneakyThrows` annotation:
+
+```java
+@SneakyThrows
+private static void sleep(Duration duration) {
+ Thread.sleep(duration);
+}
+```
+
+Since we're in an application using Java modules, we need both dependencies and the required modules. The above module declaration then becomes the following:
+
+```java
+module virtual.threads.playground {
+ requires jdk.incubator.concurrent;
+ requires org.slf4j;
+ requires static lombok;
+}
+```
+
+## 2. Why Virtual Threads?
+
+For people who already follow us, we asked the same question in the article on [Kotlin Coroutines](https://blog.rockthejvm.com/kotlin-coroutines-101/). However, it is essential to briefly introduce the problem virtual threads are trying to solve.
+
+The JVM is a multithreaded environment. As we may know, the JVM gives us an abstraction of OS threads through the type `java.lang.Thread`. **Until Project Loom, every thread in the JVM is just a little wrapper around an OS thread**. We can call the such implementation of the `java.lang.Thread` type as _platform thread_.
+
+The problem with platform threads is that they are expensive from a lot of points of view. First, they are costly to create. Whenever a platform thread is made, the **OS must allocate a large amount of memory (megabytes) in the stack to store the thread context, native, and Java call stacks**. This is due to the not resizable nature of the stack. Moreover, whenever the scheduler preempts a thread from execution, this enormous amount of memory must be moved around.
+
+As we can imagine, this is a costly operation, in space and time. In fact, the massive size of the stack frame limits the number of threads that can be created. We can reach an `OutOfMemoryError` quite easily in Java, continually instantiating new platform threads till the OS runs out of memory:
+
+```java
+private static void stackOverFlowErrorExample() {
+ for (int i = 0; i < 100_000; i++) {
+ new Thread(() -> {
+ try {
+ Thread.sleep(Duration.ofSeconds(1L));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }).start();
+ }
+}
+```
+
+The results depend on the OS and the hardware, but we can easily reach an `OutOfMemoryError` in a few seconds:
+
+```
+[0.949s][warning][os,thread] Failed to start thread "Unknown thread" - pthread_create failed (EAGAIN) for attributes: stacksize: 1024k, guardsize: 4k, detached.
+[0.949s][warning][os,thread] Failed to start the native thread for java.lang.Thread "Thread-4073"
+Exception in thread "main" java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached
+```
+
+The above example shows how we wrote concurrent programs that were constrained until now.
+
+Java has been a language that has tried to strive for simplicity since its inception. In concurrent programming, we should write programs as if they were sequential. In fact, **the more straightforward way to write concurrent programs in Java is to create a new thread for every concurrent task**. This model is called _one task per thread_.
+
+In such an approach, every thread can use its own local variable to store information. The need to share mutable states among threads, the well-known "hard part" of concurrent programming, drastically decreases. However, using such an approach, we can easily reach the limit of the number of threads we can create.
+
+As we said in the article concerning Kotlin Coroutines, many approaches have risen in recent years to overcome the above problem. The first attempt was to introduce a model of programming based on callback. For each asynchronous statement, we also give a callback to call once the statement finishes:
+
+```java
+static void callbackHell() {
+ a(aInput, resultFromA ->
+ b(resultFromA, resultFromB ->
+ c(resultFromB, resultFromC ->
+ d(resultFromC, resultFromD ->
+ System.out.printf("A, B, C, D: $resultFromA, $resultFromB, $resultFromC, $resultFromD")))));
+}
+```
+
+The above code is a simple example of callback hell. The code is not easy to read and understand. Moreover, it is not easy to write.
+
+To overcome the problems of callbacks, reactive programming, and async/await strategies were introduced.
+
+The reactive programming initiatives try to overcome the lack of thread resources by building a custom DSL to declaratively describe the data flow and let the framework handle concurrency. However, DSL is tough to understand and use, losing the simplicity Java tries to give us.
+
+Also, the async/await approach, such as Kotlin coroutines, has its own problems. Even though it aims to model the _one task per thread_ approach, it can't rely on any native JVM construct. For example, Kotlin coroutines based the whole story on _suspending functions_, i.e., functions that can suspend a coroutine. However, the suspension is wholly based upon non-blocking IO, which we can achieve using libraries based on Netty, but not every task can be expressed in terms of non-blocking IO. Ultimately, we must divide our program into two parts: one based on non-blocking IO (suspending functions) and one that does not. This is a challenging task; it takes work to do it correctly. Moreover, we lose again the simplicity we want in our programs.
+
+The above are reasons why the JVM community is looking for a better way to write concurrent programs. Project Loom is one of the attempts to solve the problem. So, let's introduce the first brick of the project: _virtual threads_.
+
+## 3. How to Create a Virtual Thread
+
+As we said, virtual threads are a new type of thread that tries to overcome the resource limitation problem of platform threads. They are an alternate implementation of the `java.lang.Thread` type, which **stores the stack frames in the heap (garbage-collected memory) instead of the stack**.
+
+Therefore, the initial memory footprint of a virtual thread tends to be very small, a few hundred bytes instead of megabytes. In fact, the stack chunk can resize at every moment. So, we don't need to allocate a gazillion of memory to fit every possible use case.
+
+Creating a new virtual thread is very easy. We can use the new factory method `ofVirtual` on the `java.lang.Thread` type. Let's first define a utility function to create a virtual thread with a given name:
+
+```java
+private static Thread virtualThread(String name, Runnable runnable) {
+ return Thread.ofVirtual()
+ .name(name)
+ .start(runnable);
+}
+```
+
+We'll use the same example in the Kotlin Coroutine article to show how virtual threads work. Let's describe our morning routine. Every morning, we take a bath:
+
+```java
+static Thread bathTime() {
+ return virtualThread(
+ "Bath time",
+ () -> {
+ log("I'm going to take a bath");
+ sleep(Duration.ofMillis(500L));
+ log("I'm done with the bath");
+ });
+}
+```
+
+Another task that we do is to boil some water to make tea:
+
+```java
+static Thread boilingWater() {
+ return virtualThread(
+ "Boil some water",
+ () -> {
+ log("I'm going to boil some water");
+ sleep(Duration.ofSeconds(1L));
+ log("I'm done with the water");
+ });
+}
+```
+
+Fortunately, we can race the two tasks to speed up the process and go to work earlier:
+
+```java
+@SneakyThrows
+static void concurrentMorningRoutine() {
+ var bathTime = bathTime();
+ var boilingWater = boilingWater();
+ bathTime.join();
+ boilingWater.join();
+}
+```
+
+We joined both virtual threads, so we can be sure that the `main` thread will not terminate before the two virtual threads. Let's run the program:
+
+```
+08:34:46.217 [boilWater] INFO in.rcard.virtual.threads.App - VirtualThread[#21,boilWater]/runnable@ForkJoinPool-1-worker-1 | I'm going to take a bath
+08:34:46.218 [boilWater] INFO in.rcard.virtual.threads.App - VirtualThread[#23,boilWater]/runnable@ForkJoinPool-1-worker-2 | I'm going to boil some water
+08:34:46.732 [bath-time] INFO in.rcard.virtual.threads.App - VirtualThread[#21,boilWater]/runnable@ForkJoinPool-1-worker-2 | I'm done with the bath
+08:34:47.231 [boilWater] INFO in.rcard.virtual.threads.App - VirtualThread[#23,boilWater]/runnable@ForkJoinPool-1-worker-2 | I'm done with the water
+```
+
+The output is what we expected. The two virtual threads run concurrently, and the `main` thread waits for them to terminate. We'll explain all the information printed by the log in a while. For now, let's focus solely on thread name and execution interleaving.
+
+Besides the factory method, we can use a new implementation of the `java.util.concurrent.ExecutorService` tailored on virtual threads, called `java.util.concurrent.ThreadPerTaskExecutor`. Its name is quite evocative. It creates a new virtual thread for every task submitted to the executor:
+
+```java
+@SneakyThrows
+static void concurrentMorningRoutineUsingExecutors() {
+ try (var executor = Executors.newVirtualThreadPerTaskExecutor()) {
+ var bathTime =
+ executor.submit(
+ () -> {
+ log("I'm going to take a bath");
+ sleep(Duration.ofMillis(500L));
+ log("I'm done with the bath");
+ });
+ var boilingWater =
+ executor.submit(
+ () -> {
+ log("I'm going to boil some water");
+ sleep(Duration.ofSeconds(1L));
+ log("I'm done with the water");
+ });
+ bathTime.get();
+ boilingWater.get();
+ }
+}
+```
+
+The way we start threads is a little different since we're using the `ExecutorService`. Every call to the `submit` method requires a `Runnable` or a `Callable` instance. The `submit` returns a `Future` instance that we can use to join the underlying virtual thread.
+
+The output is more or less the same as before:
+
+```
+08:42:09.164 [] INFO in.rcard.virtual.threads.App - VirtualThread[#21]/runnable@ForkJoinPool-1-worker-1 | I'm going to take a bath
+08:42:09.164 [] INFO in.rcard.virtual.threads.App - VirtualThread[#23]/runnable@ForkJoinPool-1-worker-2 | I'm going to boil some water
+08:42:09.676 [] INFO in.rcard.virtual.threads.App - VirtualThread[#21]/runnable@ForkJoinPool-1-worker-2 | I'm done with the bath
+08:42:10.175 [] INFO in.rcard.virtual.threads.App - VirtualThread[#23]/runnable@ForkJoinPool-1-worker-2 | I'm done with the water
+```
+
+As we can see, threads created this way do not have a name, and debugging errors without a name can be difficult. We can overcome this problem just by using the `newThreadPerTaskExecutor` factory method that takes a `ThreadFactory` as a parameter:
+
+```java
+@SneakyThrows
+static void concurrentMorningRoutineUsingExecutorsWithName() {
+ final ThreadFactory factory = Thread.ofVirtual().name("routine-", 0).factory();
+ try (var executor = Executors.newThreadPerTaskExecutor(factory)) {
+ var bathTime =
+ executor.submit(
+ () -> {
+ log("I'm going to take a bath");
+ sleep(Duration.ofMillis(500L));
+ log("I'm done with the bath");
+ });
+ var boilingWater =
+ executor.submit(
+ () -> {
+ log("I'm going to boil some water");
+ sleep(Duration.ofSeconds(1L));
+ log("I'm done with the water");
+ });
+ bathTime.get();
+ boilingWater.get();
+ }
+}
+```
+
+A `ThreadFactory` is a factory that creates threads with the same configuration. In our case, we give the prefix `routine-` to the name of the threads, and we start the counter from 0. The output is the same as before, but now we can see the name of the threads:
+
+```
+08:44:35.390 [routine-1] INFO in.rcard.virtual.threads.App - VirtualThread[#23,routine-1]/runnable@ForkJoinPool-1-worker-2 | I'm going to boil some water
+08:44:35.390 [routine-0] INFO in.rcard.virtual.threads.App - VirtualThread[#21,routine-0]/runnable@ForkJoinPool-1-worker-1 | I'm going to take a bath
+08:44:35.900 [routine-0] INFO in.rcard.virtual.threads.App - VirtualThread[#21,routine-0]/runnable@ForkJoinPool-1-worker-1 | I'm done with the bath
+08:44:36.399 [routine-1] INFO in.rcard.virtual.threads.App - VirtualThread[#23,routine-1]/runnable@ForkJoinPool-1-worker-1 | I'm done with the water
+```
+
+Now that we know how to create virtual threads let's see how they work.
+
+## 4. How Virtual Threads Work
+
+How do virtual threads work? The figure below shows the relationship between virtual threads and platform threads:
+
+
+
+**The JVM maintains a pool of platform threads**, created and maintained by a dedicated `ForkJoinPool`. Initially, the number of platform threads equals the number of CPU cores, and it cannot increase more than 256.
+
+For each created virtual thread, the JVM schedules its execution on a platform thread, temporarily copying the stack chunk for the virtual thread from the heap to the stack of the platform thread. **We said that the platform thread becomes the carrier thread of the virtual thread**.
+
+The logs we've seen so far showed us precisely the above situation. Let's analyze one of them:
+
+```
+08:44:35.390 [routine-1] INFO in.rcard.virtual.threads.App - VirtualThread[#23,routine-1]/runnable@ForkJoinPool-1-worker-2 | I'm going to boil some water
+```
+
+The exciting part is on the left side of the `|` character. The first part identifies the virtual thread in execution: `VirtualThread[#23,routine-1]` reports the thread identifier, the `#23` part, and the thread name. Then, we have the indication on which carrier thread the virtual thread executes: `ForkJoinPool-1-worker-2` represents the platform thread called `worker-2` of the default `ForkJoinPool`, called `ForkJoinPool-1`.
+
+**The first time the virtual thread blocks on a blocking operation, the carrier thread is released**, and the stack chunk of the virtual thread is copied back to the heap. This way, the carrier thread can execute any other eligible virtual threads. Once the blocked virtual thread finishes the blocking operation, the scheduler schedules it again for execution. The execution can continue on the same carrier thread or a different one.
+
+We can easily see that the number of available carrier threads is equal to the number of CPU cores by default running a program that creates and starts a number of virtual threads greater than the number of cores. On a Mac, you can retrieve the number of cores by running the following command:
+
+```bash
+sysctl hw.physicalcpu hw.logicalcpu
+```
+
+We are interested in the second value, which counts the number of logical cores. On my machine, I have 2 physical cores and 4 logical cores. Let's define a function to retrieve the number of logical cores in Java:
+
+```java
+static int numberOfCores() {
+ return Runtime.getRuntime().availableProcessors();
+}
+```
+
+Then, we can create a program that makes the desired number of virtual threads, i.e., the number of logical cores plus one:
+
+```java
+static void viewCarrierThreadPoolSize() {
+ final ThreadFactory factory = Thread.ofVirtual().name("routine-", 0).factory();
+ try (var executor = Executors.newThreadPerTaskExecutor(factory)) {
+ IntStream.range(0, numberOfCores() + 1)
+ .forEach(i -> executor.submit(() -> {
+ log("Hello, I'm a virtual thread number " + i);
+ sleep(Duration.ofSeconds(1L));
+ }));
+ }
+}
+```
+
+We expect the 5 virtual threads to be executed on 4 carrier threads, and one of the carrier threads should be reused at least once. Running the program, we can see that our hypothesis is correct:
+
+```
+08:44:54.849 [routine-0] INFO in.rcard.virtual.threads.App - VirtualThread[#21,routine-0]/runnable@ForkJoinPool-1-worker-1 | Hello, I'm a virtual thread number 0
+08:44:54.849 [routine-1] INFO in.rcard.virtual.threads.App - VirtualThread[#23,routine-1]/runnable@ForkJoinPool-1-worker-2 | Hello, I'm a virtual thread number 1
+08:44:54.849 [routine-2] INFO in.rcard.virtual.threads.App - VirtualThread[#24,routine-2]/runnable@ForkJoinPool-1-worker-3 | Hello, I'm a virtual thread number 2
+08:44:54.855 [routine-4] INFO in.rcard.virtual.threads.App - VirtualThread[#26,routine-4]/runnable@ForkJoinPool-1-worker-4 | Hello, I'm a virtual thread number 4
+08:44:54.849 [routine-3] INFO in.rcard.virtual.threads.App - VirtualThread[#25,routine-3]/runnable@ForkJoinPool-1-worker-4 | Hello, I'm a virtual thread number 3
+```
+
+There are four carrier threads, `ForkJoinPool-1-worker-1`, `ForkJoinPool-1-worker-2`, `ForkJoinPool-1-worker-3`, and `ForkJoinPool-1-worker-4`, and the `ForkJoinPool-1-worker-4` is reused twice. Awesome!
+
+The above log should ring a bell in the astute reader. How the JVM schedules virtual threads on their carrier threads? Is there any preemption? Does the JVM use cooperative scheduling instead? Let's answer these questions in the next session.
+
+## 5. The Scheduler and Cooperative Scheduling
+
+Virtual threads are scheduled using a FIFO queue consumed by a dedicated `ForkJoinPool`. The default scheduler is defined in the `java.lang.VirtualThread` class:
+
+```java
+// SDK code
+final class VirtualThread extends BaseVirtualThread {
+ private static final ForkJoinPool DEFAULT_SCHEDULER = createDefaultScheduler();
+
+ // Omissis
+
+ private static ForkJoinPool createDefaultScheduler() {
+ // Omissis
+ int parallelism, maxPoolSize, minRunnable;
+ String parallelismValue = System.getProperty("jdk.virtualThreadScheduler.parallelism");
+ String maxPoolSizeValue = System.getProperty("jdk.virtualThreadScheduler.maxPoolSize");
+ String minRunnableValue = System.getProperty("jdk.virtualThreadScheduler.minRunnable");
+ // Omissis
+ return new ForkJoinPool(parallelism, factory, handler, asyncMode,
+ 0, maxPoolSize, minRunnable, pool -> true, 30, SECONDS);
+ }
+}
+```
+
+Configuring the pool dedicated to carrier threads is possible using the above system properties. The default pool size (parallelism) equals the number of CPU cores, and the maximum pool size is at most 256. The minimum number of core threads not blocked allowed is half the pool size.
+
+In Java, **virtual threads implement cooperative scheduling**. As we saw for [Kotlin Coroutines](https://blog.rockthejvm.com/kotlin-coroutines-101/#6-cooperative-scheduling), it's a virtual thread that decides when to yield the execution to another virtual thread. In detail, the control is passed to the scheduler, and **the virtual thread is _unmounted_ from the carrier thread** when it reaches a blocking operation.
+
+We can empirically verify this behavior using the `sleep()` method and the above system properties. First, let's define a function creating a virtual thread that contains an infinite loop. Let's say we want to model an employee that is working hard on a task:
+
+```java
+static Thread workingHard() {
+ return virtualThread(
+ "Working hard",
+ () -> {
+ log("I'm working hard");
+ while (alwaysTrue()) {
+ // Do nothing
+ }
+ sleep(Duration.ofMillis(100L));
+ log("I'm done with working hard");
+ });
+}
+```
+
+As we can see, the IO operation, the `sleep()` method, is after the infinite loop. We also defined an `alwaysTrue()` function, which returns `true` and allows us to write an infinite loop without using the `while (true)` construct that is not permitted by the compiler.
+
+Then, we define a function to let our employees take a break:
+
+```java
+static Thread takeABreak() {
+ return virtualThread(
+ "Take a break",
+ () -> {
+ log("I'm going to take a break");
+ sleep(Duration.ofSeconds(1L));
+ log("I'm done with the break");
+ });
+}
+```
+
+Now, we can compose the two functions and let the two thread race:
+
+```java
+@SneakyThrows
+static void workingHardRoutine() {
+ var workingHard = workingHard();
+ var takeABreak = takeABreak();
+ workingHard.join();
+ takeABreak.join();
+}
+```
+
+Before running the `workingHardRoutine()` function, we set the three system properties:
+
+```
+-Djdk.virtualThreadScheduler.parallelism=1
+-Djdk.virtualThreadScheduler.maxPoolSize=1
+-Djdk.virtualThreadScheduler.minRunnable=1
+```
+
+The above settings force the scheduler to use a pool configured with only one carrier thread. Since the `workingHard` virtual thread never reaches a blocking operation, it will never yield the execution to the `takeABreak"` virtual thread. In fact, the output is the following:
+
+```
+21:28:35.702 [Working hard] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Working hard]/runnable@ForkJoinPool-1-worker-1 | I'm working hard
+--- Running forever ---
+```
+
+The `workingHard` virtual thread is never unmounted from the carrier thread, and the `takeABreak` virtual thread is never scheduled.
+
+Let's now change things to let the cooperative scheduling work. We define a new function simulating an employee that is working hard but stops working every 100 milliseconds:
+
+```java
+static Thread workingConsciousness() {
+ return virtualThread(
+ "Working consciousness,
+ () -> {
+ log("I'm working hard");
+ while (alwaysTrue()) {
+ sleep(Duration.ofMillis(100L));
+ }
+ log("I'm done with working hard");
+ });
+}
+```
+
+Now, the execution can reach the blocking operation, and the `workingHard` virtual thread can be unmounted from the carrier thread. To verify this, we can race the above thread with the `takeABreak` thread:
+
+```java
+@SneakyThrows
+static void workingConsciousnessRoutine() {
+ var workingConsciousness = workingConsciousness();
+ var takeABreak = takeABreak();
+ workingConsciousness.join();
+ takeABreak.join();
+}
+```
+
+This time, we expect the `takeABreak` virtual thread to be scheduled and executed on the only carrier thread when the `workingConsciousness` reaches the blocking operation. The output confirms our expectations:
+
+```
+21:30:51.677 [Working consciousness] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Working consciousness]/runnable@ForkJoinPool-1-worker-1 | I'm working hard
+21:30:51.682 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm going to take a break
+21:30:52.688 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm done with the break
+--- Running forever ---
+```
+
+As expected, the two virtual threads share the same carrier thread.
+
+Let's go back to the `workingHardRoutine()` function. If we change the carrier pool size to 2, we can see that both the `workingHard` and the `takeABreak` virtual threads are scheduled on the two carrier threads so they can run concurrently. The new setup is the following:
+
+```
+-Djdk.virtualThreadScheduler.parallelism=2
+-Djdk.virtualThreadScheduler.maxPoolSize=2
+-Djdk.virtualThreadScheduler.minRunnable=2
+```
+
+As we might expect, the output is the following. While the `ForkJoinPool-1-worker-1` is stuck in the infinite loop, the `ForkJoinPool-1-worker-2` is executing the `takeABreak` virtual thread:
+
+```
+21:33:43.641 [Working hard] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Working hard]/runnable@ForkJoinPool-1-worker-1 | I'm working hard
+21:33:43.641 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#24,Take a break]/runnable@ForkJoinPool-1-worker-2 | I'm going to take a break
+21:33:44.655 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#24,Take a break]/runnable@ForkJoinPool-1-worker-2 | I'm done with the break
+--- Running forever ---
+```
+
+It's worth mentioning that cooperative scheduling is helpful when working in a highly collaborative environment. Since a virtual thread releases its carrier thread only when reaching a blocking operation, cooperative scheduling and virtual threads will not improve the performance of CPU-intensive applications. The JVM already gives us a tool for those tasks: Java parallel streams.
+
+## 6. Pinned Virtual Threads
+
+We said that the JVM _mounts_ a virtual thread to a platform thread, its carrier thread, and executes it until it reaches a blocking operation. Then, the virtual thread is unmounted from the carrier thread, and the scheduler decides which virtual thread to schedule on the carrier thread.
+
+However, **there are some cases where a blocking operation doesn't unmount the virtual thread from the carrier thread**, blocking the underlying carrier thread. In such cases, we say the virtual is _pinned_ to the carrier thread. It's not an error but a behavior that limits the application's scalability. Note that if a carrier thread is pinned, the JVM can always add a new platform thread to the carrier pool if the configurations of the carrier pool allow it.
+
+Fortunately, there are only two cases in which a virtual thread is pinned to the carrier thread:
+
+- When it executes code inside a `synchronized` block or method;
+- When it calls a native method or a foreign function (i.e., a call to a native library using JNI).
+
+Let's see an example of pinned virtual thread. We want to simulate an employee that needs to go to the bathroom. The bathroom has only one WC, so the access to the toilet must be synchronized:
+
+```java
+static class Bathroom {
+ synchronized void useTheToilet() {
+ log("I'm going to use the toilet");
+ sleep(Duration.ofSeconds(1L));
+ log("I'm done with the toilet");
+ }
+}
+```
+
+Now, we define a function simulating an employee that uses the bathroom:
+
+```java
+static Bathroom bathroom = new Bathroom();
+
+static Thread goToTheToilet() {
+ return virtualThread(
+ "Go to the toilet",
+ () -> bathroom.useTheToilet());
+}
+```
+
+In the office, there are Riccardo and Daniel. Riccardo has to go to the bathroom while Daniel wants a break. Since they're working on different issues, they could complete their task concurrently. Let's define a function that tries to execute Riccardo and Daniel concurrently:
+
+```java
+@SneakyThrows
+static void twoEmployeesInTheOffice() {
+ var riccardo = goToTheToilet();
+ var daniel = takeABreak();
+ riccardo.join();
+ daniel.join();
+}
+```
+
+To see the effect of synchronization and the pinning of the associated `riccardo` virtual thread, we limit the carrier pool to one thread, as we did previously. The execution of the `twoEmployeesInTheOffice` produces the following output:
+
+```
+16:29:05.548 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm going to use the toilet
+16:29:06.558 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm done with the toilet
+16:29:06.559 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm going to take a break
+16:29:07.563 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm done with the break
+```
+
+As we can see, the tasks are entirely linearized by the JVM. As we said, the blocking `sleep` operation is inside the `synchronized` `useTheToilet` method, so the virtual thread is not unmounted. So, the `riccardo` virtual thread is pinned to the carrier thread, and the `daniel` virtual thread finds no available carrier thread to execute. In fact, it is scheduled when the `riccardo` virtual thread is done with the bathroom.
+
+It's possible to trace these situations during the execution of a program by adding a property to the run configuration:
+
+```
+-Djdk.tracePinnedThreads=full/short
+```
+
+The `full` value prints the full stack trace of the pinned virtual thread, while the `short` value prints only less information. The execution of the `twoEmployeesInTheOffice` with the above configuration set to the `short` value produces the following interesting output:
+
+```
+16:29:05.548 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm going to use the toilet
+Thread[#22,ForkJoinPool-1-worker-1,5,CarrierThreads]
+ virtual.threads.playground/in.rcard.virtual.threads.App$Bathroom.useTheToilet(App.java:188) <== monitors:1
+16:29:06.558 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm done with the toilet
+16:29:06.559 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm going to take a break
+16:29:07.563 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm done with the break
+```
+
+As we guessed, the `riccardo` virtual thread was pinned to its carrier thread. We can also see the name of the carrier thread here. Amazing.
+
+We can change the configuration of the carrier pool to allow the JVM to add a new carrier thread to the pool when needed:
+
+```
+-Djdk.virtualThreadScheduler.parallelism=1
+-Djdk.virtualThreadScheduler.maxPoolSize=2
+-Djdk.virtualThreadScheduler.minRunnable=1
+```
+
+We also removed the property `jdk.tracePinnedThreads` to avoid printing the pinned stacktrace. Execution with the new configuration produces the following output:
+
+```
+16:32:05.235 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm going to use the toilet
+16:32:05.235 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-2 | I'm going to take a break
+16:32:06.243 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm done with the toilet
+16:32:06.243 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-2 | I'm done with the break
+```
+
+The JVM added a new carrier thread to the pool when it found no carrier thread. So the `daniel` virtual thread is scheduled on the new carrier thread, executing concurrently and interleaving the two logs.
+
+Even though soon also `synchronized` blocks will probably unmount a virtual thread from its carrier thread, it is better to migrate those blocks to the `Lock` API, using `java.util.concurrent.locks.ReentrantLock`. Such locks don't pin the virtual thread, making the cooperative scheduling work again.
+
+Let's create a version of our `Bathroom` class using the `Lock` API:
+
+```java
+static class Bathroom {
+ private final Lock lock = new ReentrantLock();
+
+ @SneakyThrows
+ void useTheToiletWithLock() {
+ if (lock.tryLock(10, TimeUnit.SECONDS)) {
+ try {
+ log("I'm going to use the toilet");
+ sleep(Duration.ofSeconds(1L));
+ log("I'm done with the toilet");
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+}
+```
+
+Now, let's change the previous functions to use this new version of the `Bathroom` class:
+
+```java
+static Thread goToTheToiletWithLock() {
+ return virtualThread("Go to the toilet", () -> bathroom.useTheToiletWithLock());
+}
+
+@SneakyThrows
+static void twoEmployeesInTheOfficeWithLock() {
+ var riccardo = goToTheToiletWithLock();
+ var daniel = takeABreak();
+ riccardo.join();
+ daniel.join();
+}
+```
+
+The execution of the `twoEmployeesInTheOfficeWithLock` produces the expected output, which shows the two threads running concurrently:
+
+```
+16:35:58.921 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-2 | I'm going to take a break
+16:35:58.921 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-1 | I'm going to use the toilet
+16:35:59.932 [Take a break] INFO in.rcard.virtual.threads.App - VirtualThread[#23,Take a break]/runnable@ForkJoinPool-1-worker-1 | I'm done with the break
+16:35:59.933 [Go to the toilet] INFO in.rcard.virtual.threads.App - VirtualThread[#21,Go to the toilet]/runnable@ForkJoinPool-1-worker-2 | I'm done with the toilet
+```
+
+We can run the above method also with the `jdk.tracePinnedThreads` property set to see that no thread is pinned to its carrier thread during the execution.
+
+## 7. `ThreadLocal` and Thread Pools
+
+When using threads before Java 19 and Project Loom, creating a thread using the constructor was relatively uncommon. Instead, we preferred to use a thread pool or an executor service configured with a thread pool. In fact, those threads were what we now call platform threads, and the reason was that creating such threads was quite expensive operation.
+
+As we said at the beginning of this article, with virtual threads, it's not the case anymore. Creating a virtual thread is very cheap, both in space and time. Also, **they were designed with the idea of using a different virtual thread for each request**. So, it's worthless to use a thread pool or an executor service to create virtual threads.
+
+As for `ThreadLocal`, the possible high number of virtual threads created by an application is why using `ThreadLocal` may not be a good idea.
+
+What is a `ThreadLocal`? A `ThreadLocal` is a construct that allows us to store data accessible only by a specific thread. Let's see an example. First of all, we want to create a `ThreadLocal` that holds a `String`:
+
+```java
+static ThreadLocal context = new ThreadLocal<>();
+```
+Then, we create two different platform threads that use both the `ThreadLocal`:
+
+```java
+@SneakyThrows
+static void platformThreadLocal() {
+ var thread1 = Thread.ofPlatform().name("thread-1").start(() -> {
+ context.set("thread-1");
+ sleep(Duration.ofSeconds(1L));
+ log("Hey, my name is " + context.get());
+ });
+ var thread2 = Thread.ofPlatform().name("thread-2").start(() -> {
+ context.set("thread-2");
+ sleep(Duration.ofSeconds(1L));
+ log("Hey, my name is " + context.get());
+ });
+ thread1.join();
+ thread2.join();
+}
+```
+
+If we run the above function, the output is:
+
+```text
+14:57:05.334 [thread-2] INFO in.rcard.virtual.threads.App - Thread[#22,thread-2,5,main] | Hey, my name is thread-2
+14:57:05.334 [thread-1] INFO in.rcard.virtual.threads.App - Thread[#21,thread-1,5,main] | Hey, my name is thread-1
+```
+
+As we can see, each thread stores a different value in the `ThreadLocal`, which is not accessible to other threads. The thread called `thread-1` retrieves the value `thread-1` from the `ThreadLocal`; The thread `thread-2` retrieves the value `thread-2` instead. There is no race condition at all.
+
+The same properties of `ThreadLocal` still stand when we speak about virtual threads. In fact, we can replicate the same example above using virtual threads, and the result will be the same:
+
+```java
+@SneakyThrows
+static void virtualThreadLocal() {
+ var virtualThread1 = Thread.ofVirtual().name("thread-1").start(() -> {
+ context.set("thread-1");
+ sleep(Duration.ofSeconds(1L));
+ log("Hey, my name is " + context.get());
+ });
+ var virtualThread2 = Thread.ofVirtual().name("thread-2").start(() -> {
+ context.set("thread-2");
+ sleep(Duration.ofSeconds(1L));
+ log("Hey, my name is " + context.get());
+ });
+ virtualThread1.join();
+ virtualThread2.join();
+}
+```
+
+As we might expect, the output is very similar to the previous one:
+
+```text
+15:08:37.142 [thread-1] INFO in.rcard.virtual.threads.App - VirtualThread[#21,thread-1]/runnable@ForkJoinPool-1-worker-1 | Hey, my name is thread-1
+15:08:37.142 [thread-2] INFO in.rcard.virtual.threads.App - VirtualThread[#23,thread-2]/runnable@ForkJoinPool-1-worker-2 | Hey, my name is thread-2
+```
+
+Nice. So, is it a good idea to use `ThreadLocal` with virtual threads? Well, you now need to be careful. The reason is that we can have a huge number of virtual threads, and each virtual thread will have its own `ThreadLocal`. This means that **the memory footprint of the application may quickly become very high**. Moreover, the `ThreadLocal` will be useless in a one-thread-per-request scenario since data won't be shared between different requests.
+
+However, some scenarios could be help use something similar to `ThreadLocal`. For this reason, Java 20 will introduce [scoped values](https://openjdk.org/jeps/429), which enable the sharing of immutable data within and across threads. However, this is a topic for another article.
+
+## 8. Some Virtual Threads Internals
+
+In this section, we'll introduce the implementation of continuation in Java virtual threads. We're not going into too much detail, but we'll try to give a general idea of how the virtual threads are implemented.
+
+A virtual thread cannot run itself, but it stores the information of what must be run. In other words, **it's a pointer to the advance of an execution that can be yielded and resumed later**.
+
+The above is the definition of _continuations_. We've already seen how Kotlin coroutines implement continuations ([Kotlin Coroutines - A Comprehensive Introduction - Suspending Functions](https://blog.rockthejvm.com/kotlin-coroutines-101/#3-suspending-functions)). In that case, the Kotlin compiler generates continuation from the coroutine code. Kotlin's coroutines have no direct support in the JVM, so they are supported using code generation by the compiler.
+
+However, for virtual threads, we have the JVM support directly. So, continuations execution is implemented using a lot of native calls to the JVM, and it's less understandable when looking at the JDK code. However, we can still look at some concepts at the roots of virtual threads.
+
+As a continuation, a virtual thread is a state machine with many states. The relations among these states are summarized in the following diagram:
+
+
+
+A virtual thread is _mounted_ on its carrier thread when it is in the states colored green in the above diagram. In states colored in light blue, the virtual thread is _unmounted_ from its carrier thread. The pinned state is colored violet.
+
+We get a virtual thread in the `NEW` status when we call the `unstarted` method on the object returned by the `Thread.ofVirtual()` method. The core information is mainly in the `java.lang.VirtualThread` class. At the core, the JVM calls the `VirtualThread`constructor:
+
+```java
+// JDK core code
+VirtualThread(Executor scheduler, String name, int characteristics, Runnable task) {
+ super(name, characteristics, /*bound*/ false);
+ Objects.requireNonNull(task);
+ // choose scheduler if not specified
+ if (scheduler == null) {
+ Thread parent = Thread.currentThread();
+ if (parent instanceof VirtualThread vparent) {
+ scheduler = vparent.scheduler;
+ } else {
+ scheduler = DEFAULT_SCHEDULER;
+ }
+ }
+ this.scheduler = scheduler;
+ this.cont = new VThreadContinuation(this, task);
+ this.runContinuation = this::runContinuation;
+}
+```
+
+As we can see, a scheduler is chosen if not specified. The default scheduler is the one we described in the previous section. After that, a continuation is created, which is a `VThreadContinuation` object. This object is the one that stores the information of what has to be run as a `Runnable` object:
+
+```java
+// JDK core code
+private static class VThreadContinuation extends Continuation {
+ VThreadContinuation(VirtualThread vthread, Runnable task) {
+ super(VTHREAD_SCOPE, () -> vthread.run(task));
+ }
+ @Override
+ protected void onPinned(Continuation.Pinned reason) {
+ if (TRACE_PINNING_MODE > 0) {
+ boolean printAll = (TRACE_PINNING_MODE == 1);
+ PinnedThreadPrinter.printStackTrace(System.out, printAll);
+ }
+ }
+}
+```
+
+The above code also shows how the `jdk.tracePinnedThreads` flag works. The `VTHREAD_SCOPE` is a `ContinuationScope` object, a class used to group continuations. In other words, it's a way to group continuations related to each other. In our case, we have only one `ContinuationScope` object, the `VTHREAD_SCOPE` object. This object is used to group all the virtual threads.
+
+Last, the method sets the `runContinuation` field, a `Runnable` object used to run the continuation. This method is called when the virtual thread is started.
+
+Once we call the `start` method, the virtual thread is moved to the `STARTED` status:
+
+```java
+// JDK core code
+@Override
+void start(ThreadContainer container) {
+ if (!compareAndSetState(NEW, STARTED)) {
+ throw new IllegalThreadStateException("Already start
+ }
+ // Omissis
+ try {
+ // Omissis
+ // submit task to run thread
+ submitRunContinuation();
+ started = true;
+ } finally {
+ // Omissis
+ }
+}
+```
+
+The `submitRunContinuation()` is the method scheduling the `runContinuation` runnable to the virtual thread scheduler:
+
+```java
+// JDK core code
+private void submitRunContinuation(boolean lazySubmit) {
+ try {
+ if (lazySubmit && scheduler instanceof ForkJoinPool pool) {
+ pool.lazySubmit(ForkJoinTask.adapt(runContinuation));
+ } else {
+ scheduler.execute(runContinuation);
+ }
+ } catch (RejectedExecutionException ree) {
+ // Omissis
+ }
+}
+```
+
+The execution of the `runContinuation` runnable moves the virtual thread to the `RUNNING` status, both if it's in the `STARTED` status or in the `RUNNABLE` status:
+
+```java
+// JDK core code
+private void runContinuation() {
+ // Omissis
+ if (initialState == STARTED && compareAndSetState(STARTED, RUNNING)) {
+ // first run
+ firstRun = true;
+ } else if (initialState == RUNNABLE && compareAndSetState(RUNNABLE, RUNNING)) {
+ // consume parking permit
+ setParkPermit(false);
+ firstRun = false;
+ } else {
+ // not runnable
+ return;
+ }
+ // Omissis
+ try {
+ cont.run();
+ } finally {
+ // Omissis
+ }
+}
+```
+
+From this point on, the state of the virtual threads depends on the execution of the continuation, made through the method `Continuation.run()`. The method performs a lot of native calls, and it's not easy to follow the execution flow. However, the first thing it makes is to set as mounted the associated virtual thread:
+
+```java
+// JDK core code
+public final void run() {
+ while (true) {
+ mount();
+ // A lot of omissis
+ }
+}
+```
+
+Every time the virtual thread reaches a blocking point, the state of the thread is changed to `PARKING`. The reaching of a blocking point is signaled through the call of the `VirtualThread.park()` method:
+
+```java
+// JDK core code
+void park() {
+ assert Thread.currentThread() == this;
+ // complete immediately if parking permit available or interrupted
+ if (getAndSetParkPermit(false) || interrupted)
+ return;
+ // park the thread
+ setState(PARKING);
+ try {
+ if (!yieldContinuation()) {
+ // park on the carrier thread when pinned
+ parkOnCarrierThread(false, 0);
+ }
+ } finally {
+ assert (Thread.currentThread() == this) && (state() == RUNNING);
+ }
+}
+```
+
+Once in the `PARKING` state, the `yieldContinuation()` method is called. This method is the one that performs the actual parking of the virtual thread and tries to unmount the virtual thread from its carrier thread:
+
+```java
+// JDK core code
+private boolean yieldContinuation() {
+ boolean notifyJvmti = notifyJvmtiEvents;
+ // unmount
+ if (notifyJvmti) notifyJvmtiUnmountBegin(false);
+ unmount();
+ try {
+ return Continuation.yield(VTHREAD_SCOPE);
+ } finally {
+ // re-mount
+ mount();
+ if (notifyJvmti) notifyJvmtiMountEnd(false);
+ }
+}
+```
+
+The `Continuation.yield(VTHREAD_SCOPE)` call is implemented with many JVM native calls. If the method returns `true`, then the `parkOnCarrierThread`is called. This method sets the virtual threads as pinned on the carrier thread:
+
+```java
+private void parkOnCarrierThread(boolean timed, long nanos) {
+ assert state() == PARKING;
+ var pinnedEvent = new VirtualThreadPinnedEvent();
+ pinnedEvent.begin();
+ setState(PINNED);
+ try {
+ if (!parkPermit) {
+ if (!timed) {
+ U.park(false, 0);
+ } else if (nanos > 0) {
+ U.park(false, nanos);
+ }
+ }
+ } finally {
+ setState(RUNNING);
+ }
+ // consume parking permit
+ setParkPermit(false);
+ pinnedEvent.commit();
+}
+```
+
+From there, the method `VirtualThread.afterYield()` is called. This method sets the `PARKED` state to the virtual thread, and the continuation is scheduled again for execution through the method `lazySubmitRunContinuation()` and setting the state to `RUNNABLE`:
+
+```java
+// JDK core code
+private void afterYield() {
+ int s = state();
+ assert (s == PARKING || s == YIELDING) && (carrierThread == null);
+ if (s == PARKING) {
+ setState(PARKED);
+ // notify JVMTI that unmount has completed, thread is parked
+ if (notifyJvmtiEvents) notifyJvmtiUnmountEnd(false);
+ // may have been unparked while parking
+ if (parkPermit && compareAndSetState(PARKED, RUNNABLE)) {
+ // lazy submit to continue on the current thread as carrier if possible
+ lazySubmitRunContinuation();
+ }
+ } else if (s == YIELDING) { // Thread.yield
+ setState(RUNNABLE);
+ // notify JVMTI that unmount has completed, thread is runnable
+ if (notifyJvmtiEvents) notifyJvmtiUnmountEnd(false);
+ // lazy submit to continue on the current thread as carrier if possible
+ lazySubmitRunContinuation();
+ }
+}
+```
+
+This closes the circle. As we can see, it takes a lot of work to follow the life cycle of a virtual thread and its continuation. A lot of native calls are involved. We hope that the JDK team will provide better documentation of the virtual threads implementation in the future.
+
+## 9. Conclusions
+
+Finally, we come to the end of this article. In the beginning, we introduced the reason behind the introduction of virtual threads in the JVM. Then, we saw how to create and use it with some examples. We made some examples of pinned threads, and finally, we saw how some old best practices are no longer valid when using virtual threads.
+
+Project Loom is still actively under development, and there are a lot of other exciting features in it. As we said, structural concurrency and scoped values are some of them. Project Loom will be a game changer in the Java world. This article will help you better understand virtual threads and how to use them.
+
+## 10. Appendix: Maven Configuration
+
+As promised, here is the `pom.xml` file that we used to run the code in this article:
+
+```xml
+
+
+ 4.0.0
+
+ in.rcard
+ virtual-threads-playground
+ jar
+ 1.0.0-SNAPSHOT
+ Java Virtual Threads Playground
+
+
+ 19
+ 19
+ UTF-8
+
+
+
+
+ org.projectlombok
+ lombok
+ edge-SNAPSHOT
+ provided
+
+
+ ch.qos.logback
+ logback-classic
+ 1.4.5
+
+
+
+
+
+
+
+ maven-clean-plugin
+ 3.2.0
+
+
+ maven-resources-plugin
+ 3.3.0
+
+
+ maven-compiler-plugin
+ 3.10.1
+
+ 19
+ --enable-preview
+
+
+
+ maven-surefire-plugin
+ 3.0.0-M7
+
+
+ maven-jar-plugin
+ 3.2.2
+
+
+ maven-install-plugin
+ 3.0.1
+
+
+ maven-deploy-plugin
+ 3.0.0
+
+
+ maven-site-plugin
+ 4.0.0-M3
+
+
+ maven-project-info-reports-plugin
+ 3.4.0
+
+
+
+
+
+
+
+ projectlombok.org
+ https://projectlombok.org/edge-releases
+
+
+
+
+```
diff --git a/_posts/2023-04-21-security-in-http4s.md b/_posts/2023-04-21-security-in-http4s.md
new file mode 100644
index 000000000000..26f7af871d37
--- /dev/null
+++ b/_posts/2023-04-21-security-in-http4s.md
@@ -0,0 +1,611 @@
+---
+title: "Configuring Http4s Security: CORS and CSRF"
+date: 2023-03-30
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "In this article, we will show you how to set up an Http4s server so that it correctly protects against cross-origin requests and CSRF."
+toc: true
+toc_label: "In this article"
+---
+
+_This article is brought to you by [Herbert Kateu](https://github.com/hkateu), a new contributor. He got started after applying for the fresh new Technical Writer position at our new [Job Board](https://jobs.rockthejvm.com/)!_
+
+## 1. Introduction
+
+With the growing number of cyber-attacks ever increasing, there's a growing need for security in the applications we build.
+Http4s comes with several easily configurable security features and in this article, we will cover the two most common, CORS and CSRF.
+If you are not familiar with http4s, a good introduction to the library can be found [here](https://blog.rockthejvm.com/http4s-tutorial/)
+
+## 2. Setting Up
+
+To run the code in this article, we'll need to add the following dependencies in our project build:
+
+```scala
+libraryDependencies ++= Seq(
+ "org.typelevel" %% "cats-core" % "2.8.0",
+ "org.typelevel" %% "cats-effect" % "3.3.14"
+ "org.http4s" %% "http4s-dsl" % "0.23.16"
+ "org.http4s" %% "http4s-ember-server" % "0.23.16"
+)
+
+```
+
+We'll also need these two programs installed to follow along.
+The [curl](https://curl.se/) command line tool for sending requests in the terminal.
+The [serve](https://www.npmjs.com/package/serve) npm package, a lightweight server for static pages.
+
+## 3. Cross-Origin Resource Sharing
+
+Many developers have come across the CORS acronym at some point in their careers, but what does it mean exactly? CORS stands for Cross-Origin Resource Sharing, it's a technique used by browsers to ensure secure requests and data transfers from any origin other than its own. Here's how this would work in 3 simple steps.
+
+1. `img.com` needs to load images on its landing page but these images are hosted on `imagebucket.com`
+2. When someone visits `img.com`, his/her browser will send a request for images to `imagebucket.com`, this is called a cross-origin request.
+3. If `imagebucket.com` setup cross-origin resource sharing to include `img.com`, then the browser will proceed and load these requests, otherwise the request will be canceled and the images will fail to load.
+
+Imagine for a second that CORS didn't exist, malicious sites could easily request and acquire information from any site by making cross-origin requests. Typically a server should contain a list of approved sites to which cross-origin resource sharing is approved, any requests made from sites outside this list should be denied.
+
+## 4. CORS and Http4s
+
+Http4s provides CORS as part of it's `middleware`, the `CORS` package comes with a number of methods that help in implementing CORS within Http4s.
+Let's create our minimalistic server to show how CORS works. The code that follows is written in Scala 3, but can also work with Scala 2:
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+
+object CorsExample extends IOApp {
+ val imageService = HttpRoutes.of[IO]{
+ case GET -> Root / "image" / name =>
+ Ok(s"Processing image: $name." )
+ }.orNotFound
+
+ override def run(args: List[String]): IO[ExitCode] = ???
+}
+```
+
+Here we implement our `imageService` that returns an image when provided a name, it receives a `GET` request and responds with a message telling the user that the image is being returned.
+Once the service is created, we will need to bind it to a port number where the requests will be routed, for this, we will use `ember` server:
+
+```scala
+import org.http4s.ember.server.EmberServerBuilder
+import com.comcast.ip4s.*
+
+val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(imageService)
+ .build
+
+override def run(args: List[String]): IO[ExitCode] = server.use(_ => IO.never).as(ExitCode.Success)
+```
+
+We use `EmberServerBuilder` to build our server on `localhost`, port `8080`. We also configured the server to run indefinitely using `IO.never` on the `run()` method.
+
+Now let's test the server with `curl` and see what kind of result we get:
+
+```bash
+curl -v http://localhost:8080/image/personal.jpg
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /image/personal.jpg HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Thu, 13 Apr 2023 18:28:44 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 31
+<
+* Connection #0 to host localhost left intact
+Processing image: personal.jpg.⏎
+
+```
+
+When `curl` is run with the `-v` or `--verbose` flag, it displays the headers that are sent as well as what is returned. Looking at the bottom of the response from the server, we confirm that we receive the appropriate response, `Processing image: personal.jpg`.
+
+Scrutinizing the output reveals that there's no `Origin` Header. This means that CORS has not yet been implemented. Let's fix that now:
+
+```scala
+import org.http4s.server.middleware.CORS
+
+val corsService = CORS.policy
+ .withAllowOriginAll(imageService)
+
+val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(corsService)
+ .build
+```
+
+Our first implementation of CORS uses the `withAllowOriginAll()` method on the `CORS.policy` object, this method opens the server to process requests from any origin as seen in the output below:
+
+```bash
+curl -v http://localhost:8080/image/personal.jpg -H 'Origin: http://img.com'
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /image/personal.jpg HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+> Origin: http://img.com
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Thu, 13 Apr 2023 18:58:50 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 31
+< Access-Control-Allow-Origin: *
+<
+* Connection #0 to host localhost left intact
+Processing image: personal.jpg.⏎
+
+```
+
+`Curl` can pass headers when sending requests through the `-H` flag, in our example, we passed the `Origin` header as `http://img.com`.
+Within the response, we see a new header added, `Access-Control-Allow-Origin` which is now displayed with a value `*`, the asterisk signifies that requests from all origins will be accepted by our server.
+It's important to note that the CORS configuration won't prevent non-browser requests from going through. CORS is browser specific. The server sends CORS specific headers to the browser which it uses to either block or load requests. The `Origin` header is added automatically by the browser whenever requests are sent, however since we are using `curl`, we add it in manually.
+
+With CORS implemented, we realize that even malicious sites can make requests for our `personal.jpg` image.
+
+We can specify the sites that can access our server by replacing the `withAllowOriginAll()` method with the `withAllowOriginHost()` method. This would be implemented as follows:
+
+```scala
+import org.http4s.headers.Origin
+
+val corsService = CORS.policy
+ .withAllowOriginHost(Set(
+ Origin.Host(Uri.Scheme.http, Uri.RegName("localhost"), Some(5000)),
+ ))
+ .apply(imageService)
+```
+
+The `withAllowOriginHost` method takes a Set of URLs described as `Origin` objects, the `Host` method takes a URI scheme as either `http` or `https`, a hostname and port number. Here we configured our server to only accept requests from `localhost:5000`. Non-browser requests such as those from `curl` will always go through since there's no mechanism to restrict access based on the type of CORS header returned. Let's prove this:
+
+```bash
+curl -v http://localhost:8080/image/personal.jpg -H 'Origin:http://img.com'
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /image/personal.jpg HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+> Origin: http://img.com
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Tue, 18 Apr 2023 12:42:05 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 31
+< Vary: Origin
+<
+* Connection #0 to host localhost left intact
+Processing image: personal.jpg.⏎
+```
+
+Instead of the `Access-Control-Allow-Origin`, we now receive a `Vary` header, with the value `Origin`. This means that access to our server resources is now restricted for some origins. However, access should have been restricted only to `localhost:5000` but our `personal.jpg` image still gets processed because `curl` doesn't care about CORS headers.
+
+Knowing this let's send the request through a browser. First, we create a folder to store our `HTML` file called `testFolder` and within it, create a `test.html` page to which we save the following code:
+
+```html
+
+
+
+ Fetch Image
+
+```
+
+We embed our request code in JavaScript within the `
+```
+
+In this case, the form will automatically be submitted as soon as the HTML has loaded.
+
+### 5.2 The solution
+
+If `photos.com` implemented CSRF protection correctly, this is how the above attack would be stopped.
+
+1. Whenever an HTML form is used to send `POST` requests authorizing transfer photos from `photos.com`, a `CSRF token` must be inserted within a hidden field in the form.
+2. Once the `POST` request is received by the `photos.com` server, it checks and verifies the `CSRF token` against its database. If the token is present and valid, the request would go through, however, if the token is missing or wrong, the transfer request would be rejected.
+
+## 6. CSRF protection in Http4s
+
+The `CSRF` module is also part of the Http4s `middleware` package. Let's start by creating the server we shall use in this section:
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.ember.server.EmberServerBuilder
+import com.comcast.ip4s.*
+
+object csrfExample extends IOApp {
+ val service = HttpRoutes.of[IO]{
+ case GET -> Root / "testing" =>
+ Ok(s"Testing" )
+ }.orNotFound
+
+ override def run(args: List[String]): IO[ExitCode] = server.use(_ => IO.never).as(ExitCode.Success)
+}
+```
+
+If we test our server, we should receive the string `Testing`:
+
+```bash
+curl -v http://localhost:8080/testing
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /testing HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Thu, 13 Apr 2023 21:31:27 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 7
+<
+* Connection #0 to host localhost left intact
+Testing⏎
+```
+
+From the output, we see our expected string.
+We need a way to receive post requests from the client. To implement this, we shall add another route to the photoService:
+
+```scala
+val photoService = HttpRoutes.of[IO]{
+ case GET -> Root / "testing" =>
+ Ok(s"Testing" )
+ case POST -> Root / "photos" =>
+ Ok("Processing")
+}.orNotFound
+```
+
+```bash
+curl -v -X POST http://localhost:8080/photos
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> POST /photos HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Thu, 13 Apr 2023 21:51:47 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 10
+<
+* Connection #0 to host localhost left intact
+Processing⏎
+```
+
+`Curl` handles `POST` requests by passing the `-X POST` flag as one of its arguments. In this case, we did not provide a payload to keep the example simple. From the result, we receive the expected string `Processing`.
+Next, we'll implement the `CSRF` `middleware` into our server:
+
+```scala
+import cats.effect.unsafe.IORuntime
+import org.http4s.server.middleware.CSRF
+
+implicit override val runtime: IORuntime = IORuntime.global
+val cookieName = "csrf-token"
+val token = CSRF.generateSigningKey[IO]().unsafeRunSync()
+
+val defaultOriginCheck: Request[IO] => Boolean =
+ CSRF.defaultOriginCheck[IO](_,"localhost",Uri.Scheme.http, None)
+val csrfBuilder = CSRF[IO,IO](token,defaultOriginCheck)
+
+val csrf = csrfBuilder
+ .withCookieName(cookieName)
+ .withCookieDomain(Some("localhost"))
+ .withCookiePath(Some("/"))
+ .build
+
+val csrfServer = csrf.validate()(photoService)
+
+ val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(csrfServer)
+ .build
+```
+
+We start by generating our csrf `token` which is done by running `CSRF.generateSigningKey[IO]().unsafeRunSync()`, this gives us a value of type `SecretKey`.
+The `defaultOriginCheck` is a function from `Request[IO] => Boolean` used to check the origin of the request, in our case it is `localhost`.
+The first argument to the `defaultOriginCheck` method is the request, we provided an underscore since the request will be coming through `ember` server, however, scala http4s also has a utility for creating Requests within the application.
+The third input is the `Uri` scheme which can be `http` or `https` and finally the port number which was provided as `None`.
+
+The next step is assembling the `csrfBuilder` of type `CSRFBuillder[IO,IO]`, this is done by calling the `CSRF[IO,IO]()` function which takes as arguments our `token` and the `defaultOriginCheck` function.
+The `csrfBuilder` is the heart of the CSRF implementation in http4s, in our example we provide 4 methods, first the `withCookieName()` method which takes `cookieName`, the name of the cookie as an argument, second is the `withCookieDomain()` which in our case is `localhost`, third is the `withCookiePath()` to which we provided `Some("/")`, this is the path through which we will request for the csrf token. Finally, the `build` method which bundles all these values into a `CSRF` object.
+
+The `csrfServer` now has a `validate()` method which confirms the csrf token received through the `GET` request with what it generated. We also make sure we pass the `csrfServer` to our `server` through the `withHttpApp()` method.
+
+Let's test our application and see how it responds to csrf attacks. First, we send a request without any csrf token:
+
+```bash
+curl -v -X POST http://localhost:8080/photos
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> POST /photos HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 403 Forbidden
+< Date: Thu, 13 Apr 2023 22:52:49 GMT
+< Connection: keep-alive
+< Content-Length: 0
+<
+* Connection #0 to host localhost left intact
+```
+
+We received a `403 Forbidden error`, this is a step in the right direction. In a normal front-end application, when the page loads, a request would be sent to the server to receive the csrf token, which is then placed within a form. The `withCookiePath()` method from the `csrfBuilder` contains this path which we set to `Some("/")`, let's request for our csrf token:
+
+```bash
+curl -v http://localhost:8080
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET / HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 404 Not Found
+< Date: Thu, 13 Apr 2023 23:21:24 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 9
+< Set-Cookie: csrf-token=731EB4B0C3387DA4E1A99289ADEA4BB3006640EB4D0002D5F80F54E718654C7B-1681428084141-8C1AA35A16C2E13DC8AEEB4A0F680103882B4F8B; Domain=localhost; Path=/; SameSite=Lax; HttpOnly
+<
+* Connection #0 to host localhost left intact
+Not found⏎
+```
+
+Notice that this is a `GET` request, the server sends the csrf token to the browser using the `Set-Cookie` header, this means that the csrf token is set as a cookie on the client's browser with the name `csrf-token`.
+
+Now we can send a `POST` request to the server with the csrf token in order to fetch our photos:
+
+```bash
+curl -v -X POST http://localhost:8080/photos -H "Origin:http://localhost"
+-H "X-Csrf-Token:731EB4B0C3387DA4E1A99289ADEA4BB3006640EB4D0002D5F80F54E718654C7B-1681428084141-8C1AA35A16C2E13DC8AEEB4A0F680103882B4F8B"
+--cookie "csrf-token=731EB4B0C3387DA4E1A99289ADEA4BB3006640EB4D0002D5F80F54E718654C7B-1681428084141-8C1AA35A16C2E13DC8AEEB4A0F680103882B4F8B"
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> POST /photos HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+> Cookie: csrf-token=731EB4B0C3387DA4E1A99289ADEA4BB3006640EB4D0002D5F80F54E718654C7B-1681428084141-8C1AA35A16C2E13DC8AEEB4A0F680103882B4F8B
+> Origin:http://localhost
+> X-Csrf-Token:731EB4B0C3387DA4E1A99289ADEA4BB3006640EB4D0002D5F80F54E718654C7B-1681428084141-8C1AA35A16C2E13DC8AEEB4A0F680103882B4F8B
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Thu, 13 Apr 2023 23:25:06 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 10
+< Set-Cookie: csrf-token=731EB4B0C3387DA4E1A99289ADEA4BB3006640EB4D0002D5F80F54E718654C7B-1681428306255-7C434E5B03A3AF02DFFE25726C224F107B1E3C4D; Domain=localhost; Path=/; SameSite=Lax; HttpOnly
+<
+* Connection #0 to host localhost left intact
+Processing⏎
+```
+
+CSRF is now working since we receive the `Processing` message from the server. Using `curl`, we send a `POST` request by supplying the `-X POST` flag, this is then followed by the server URL which is `http://localhost:8080/photos`. We then add the headers `Origin` and `X-Csrf-Token` with their respective values and finally, we passed the `csrf-token` cookie which was set by the server.
+
+Some gotcha's in this request, this does not apply if your requests are sent through the browser:
+If the `Origin` header is missing, the request will fail, this is because the CSRF middleware through the defaultOriginCheck method confirms the origin of the request. The browser will automatically add the Origin header, however, it's the developer's responsibility to add the Origin header for non-browser requests.
+If the cookie `csrf-token` is missing, the request will also fail. When the `csrf-token` is requested, the server will send a `Set-Cookie` header to the browser with the `csrf-token` value which sets it as a cookie, if the user visits this page again, the browser would send this cookie back to the server automatically. In the case of `curl`, we need to send this cookie back manually.
+
+Here's the full code for the csrf implementation.
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.ember.server.EmberServerBuilder
+import com.comcast.ip4s.*
+import cats.effect.unsafe.IORuntime
+import org.http4s.server.middleware.CSRF
+import javax.crypto.SecretKey
+
+object csrfExample extends IOApp {
+
+ val photoService = HttpRoutes.of[IO]{
+ case GET -> Root / "testing" =>
+ Ok(s"Testing" )
+ case POST -> Root / "photos" =>
+ Ok("Processing")
+ }.orNotFound
+
+ implicit override val runtime: IORuntime = IORuntime.global
+ val cookieName = "csrf-token"
+ val token: SecretKey = CSRF.generateSigningKey[IO]().unsafeRunSync()
+
+ val defaultOriginCheck: Request[IO] => Boolean =
+ CSRF.defaultOriginCheck[IO](_,"localhost",Uri.Scheme.http, None)
+ val csrfBuilder = CSRF[IO,IO](token,defaultOriginCheck)
+
+ val csrf = csrfBuilder
+ .withCookieName(cookieName)
+ .withCookieDomain(Some("localhost"))
+ .withCookiePath(Some("/"))
+ .build
+
+ val csrfServer = csrf.validate()(photoService)
+
+ val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(csrfServer)
+ .build
+
+ override def run(args: List[String]): IO[ExitCode] = server.use(_ => IO.never).as(ExitCode.Success)
+}
+```
+
+## 7. Conclusion
+
+In this article we went through Cross-Origin Resource Sharing, what it is, and how it's implemented. We also went through Cross-site request forgery attacks, how they occur, and how to prevent them. Both these topics have well-thought-out implementations in Http4s which we covered, and I encourage you to implement them in your servers.
diff --git a/_posts/2023-05-03-functional-error-handling-in-kotlin.md b/_posts/2023-05-03-functional-error-handling-in-kotlin.md
new file mode 100644
index 000000000000..61eac4ec4933
--- /dev/null
+++ b/_posts/2023-05-03-functional-error-handling-in-kotlin.md
@@ -0,0 +1,745 @@
+---
+title: "Functional Error Handling in Kotlin, Part 1: Absent values, Nullables, Options"
+date: 2023-04-27
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kotlin]
+excerpt: "Whether we develop using an object-oriented or functional approach, we always have the problem of handling errors. Kotlin offers a lot of different methods to do it. Here, we'll focus on strategies that deal with the error without managing its cause, i.e., nullable types and Arrow Option types."
+toc: true
+toc_label: "In this article"
+---
+
+_This article is brought to you by [Riccardo Cardin](https://github.com/rcardin). Riccardo is a proud alumnus of Rock the JVM, now a senior engineer working on critical systems written in Scala and Kotlin._
+
+If you'd like to watch the video form of this article, please enjoy:
+
+{% include video id="783eGYfsVuU" provider="youtube" %}
+
+The Kotlin language is a multi-paradigm, general-purpose programming language. Whether we develop using an object-oriented or functional approach, we always have the problem of handling errors. Kotlin offers a lot of different methods to handle errors. Still, this article will focus on the functional approaches and introduce the Arrow library. This article is the first part of a series. We'll focus on strategies that deal with the error without managing its cause, i.e., nullable types and Arrow Option types. So, without further ado, let's get started.
+
+> This article assumes you're comfortable with Kotlin. If you need to get those essential skills **as fast as possible** and with thousands of lines of code and a project under your belt, you'll love [Kotlin Essentials](https://rockthejvm.com/p/kotlin-essentials). It's a jam-packed course on **everything** you'll ever need to work with Kotlin for any platform (Android, native, backend, anything), including less-known techniques and language tricks that will make your dev life easier. Check it out [here](https://rockthejvm.com/p/kotlin-essentials).
+
+## 1. Setup
+
+Let's first create the setup we'll use throughout the article as usual. We'll use the last version of Kotlin available at the moment of writing, version 1.8.20.
+
+As we said, we will use the [Arrow](https://arrow-kt.io/) libraries. Arrow adds functional types to the Kotlin standard library. In detail, we'll use the [core](https://arrow-kt.io/docs/core/) library. Here, it is the dependency we need:
+
+```xml
+
+ io.arrow-kt
+ arrow-core
+ 1.1.5
+ pom
+
+```
+
+We'll use Maven to build the project. You'll find the complete `pom.xml` file at the end of the article.
+
+## 2. Why Exception Handling is Not Functional
+
+First, we need to understand what's wrong with the traditional approach to error handling, which is based on exceptions. We need some context to understand the problem.
+
+Imagine we want to create an application that manages a job board. First, we need a simplified model of a job:
+
+```kotlin
+data class Job(val id: JobId, val company: Company, val role: Role, val salary: Salary)
+
+@JvmInline
+value class JobId(val value: Long)
+
+@JvmInline
+value class Company(val name: String)
+
+@JvmInline
+value class Role(val name: String)
+
+@JvmInline
+value class Salary(val value: Double) {
+ operator fun compareTo(other: Salary): Int = value.compareTo(other.value)
+}
+```
+
+Then, we can simulate a database of jobs using a `Map`:
+
+```kotlin
+val JOBS_DATABASE: Map = mapOf(
+ JobId(1) to Job(
+ JobId(1),
+ Company("Apple, Inc."),
+ Role("Software Engineer"),
+ Salary(70_000.00),
+ ),
+ JobId(2) to Job(
+ JobId(2),
+ Company("Microsoft"),
+ Role("Software Engineer"),
+ Salary(80_000.00),
+ ),
+ JobId(3) to Job(
+ JobId(3),
+ Company("Google"),
+ Role("Software Engineer"),
+ Salary(90_000.00),
+ ),
+)
+```
+
+We can use a dedicated module to retrieve the job information. As a first operation, we want to retrieve a job by its id:
+
+```kotlin
+interface Jobs {
+ fun findById(id: JobId): Job
+}
+```
+
+Let's start with a trivial implementation of the `Jobs` interface:
+
+```kotlin
+class LiveJobs : Jobs {
+ override fun findById(id: JobId): Job {
+ val maybeJob: Job? = JOBS_DATABASE[id]
+ if (maybeJob != null) {
+ return maybeJob
+ } else {
+ throw NoSuchElementException("Job not found")
+ }
+ }
+}
+```
+
+When the job was not found, we threw a `NoSuchElementException`. Easy peasy.
+
+Now that we have our `Jobs` module, we can use it in a program. Let's say we want to retrieve the salary associated with a particular job. The program is straightforward, but it's enough to show the problem:
+
+```kotlin
+class JobsService(private val jobs: Jobs) {
+ fun retrieveSalary(id: JobId): Double {
+ val job = jobs.findById(id)
+ return try {
+ job.salary.value
+ } catch (e: Exception) {
+ 0.0
+ }
+ }
+}
+```
+
+Now, let's see what happens when we run the program for a job that doesn't exist:
+
+```kotlin
+fun main() {
+ val jobs: Jobs = LiveJobs()
+ val jobsService = JobsService(jobs)
+ val jobId: Long = 42
+ val salary = jobsService.retrieveSalary(JobId(jobId))
+ println("The salary of the job $jobId is $salary")
+}
+```
+
+As expected, the program crashes with the following exception:
+
+```text
+Exception in thread "main" java.util.NoSuchElementException: Job not found
+ at in.rcard.exception.LiveJobs.findById-aQvFPlM(ExceptionErrorHandling.kt:17)
+ at in.rcard.exception.JobsService.retrieveSalary-aQvFPlM(ExceptionErrorHandling.kt:24)
+ at in.rcard.MainKt.main(Main.kt:12)
+ at in.rcard.MainKt.main(Main.kt)
+```
+
+Fair enough. The exception is thrown outside the `try-catch` block and bubbles up to the `main` method.
+
+One of the main principles of functional programming is referential transparency, which states that a function should always return the same result when called with the same arguments. In other words, a function should not have any side effects. One of the results of this principle is that **an expression can be replaced with its value without changing the program's behavior**.
+
+Let's try it on our program. If we substitute the `jobs.findById(42)` expression to the `job` variable in the `retrieveSalary`method, we get the following code:
+
+```kotlin
+fun retrieveSalary(id: JobId): Double {
+ val job: Job = throw NoSuchElementException("Job not found")
+ return try {
+ job.salary.value
+ } catch (e: Exception) {
+ 0.0
+ }
+}
+```
+
+As expected, we obtained a program that crashes with the same exception. However, if we move the retrieval of the job inside the `try-catch` block, we get the following code:
+
+```kotlin
+fun retrieveSalary(id: JobId): Double {
+ return try {
+ val job = jobs.findById(id)
+ job.salary.value
+ } catch (e: Exception) {
+ 0.0
+ }
+}
+```
+
+If we execute this code in our program, we obtain a different result than the previous execution. In fact, the exception is now caught by the `try-catch` block, and the program generates the following output.
+
+```text
+The salary of the job 42 is 0.0
+```
+
+We've just proven that exceptions don't follow the substitution principle. In other words, exceptions are not referentially transparent and, for this reason, are considered a side effect. In other words, **an expression throwing exceptions can't be reasoned about without a context of execution, aka the code around the expression**. So, when we use the expression, we also lose the locality of reasoning and add a lot of cognitive loads to understand the code.
+
+Moreover, there is one more aspect we would like to change about exceptions. Let's take the signature of the `retrieveSalary` method:
+
+```kotlin
+fun retrieveSalary(id: JobId): Double
+```
+
+We expect the method to take a job id as input and return a salary as a `Double`. No reference to the exception is present in the signature. **As developers, we want the compilers to help us avoid errors**. However, in this case, we're not aware that the method can throw an exception, and the compiler can not help us in any way. The only place we become aware of the exception is during runtime execution, which is a bit late.
+
+Somebody can say that the JVM also has checked exceptions and that we can use them to avoid the problem. However, **Kotlin doesn't have checked exceptions**. Let's try to act as if it has them. If a method declares to throw a checked exception, the compiler will force us to handle it. But, **checked exceptions don't work well with higher-order functions**, which are fundamental to functional programming. In fact, if we want to use a higher-order function together with checked exceptions, we need to declare the exception in the signature of the lambda function, which is not feasible. Take the `map` function of any collection type:
+
+```kotlin
+fun map(list: List, f: (A) -> B): List
+```
+
+As we might guess, using checked exceptions for the function `f` is impossible since it must stay generic. The only possible way is to add some generic exception to the function's signature, such as an `Exception`, which is useless.
+
+So, we understood that we need a better approach to handle errors, at least in functional programming. Let's see how we can do it.
+
+## 3. Handling Errors with Nullable Types
+
+Suppose we're not interested in the cause of the error. In that case, we can model that an operation failed by returning a `null` value. In other languages returning a `null` is not considered a best practice. However, **in Kotlin, the null check is built-in the language, and the compiler can help us to avoid errors**. In Kotlin, we have nullable types. A nullable type is a type that can be either a value of the class or `null`. For example, the type `String?` is a nullable type, and it can be either a `String` or `null`.
+
+When we work with nullable types, the compiler forces us to handle the case when the value is `null`. For example, let's change our primary example, handling errors using nullable types. First, we redefined the `Jobs` interface and its implementation to use nullable types:
+
+```kotlin
+interface Jobs {
+ fun findById(id: JobId): Job?
+}
+
+class LiveJobs : Jobs {
+ override fun findById(id: JobId): Job? = try {
+ JOBS_DATABASE[id]
+ } catch (e: Exception) {
+ null
+ }
+}
+```
+
+As we said, using nullable types to handle failures means completely losing the cause of the error, which is not propagated in any way to the caller.
+
+Then, we also change the `JobsService`, and we try to use the dereference operator `.` to access the `salary` property of the `Job` object directly:
+
+```kotlin
+class JobsService(private val jobs: Jobs) {
+ fun retrieveSalary(id: JobId): Double =
+ jobs.findById(id).salary.value
+}
+```
+
+If we try to compile the code, we get the following error:
+
+```text
+Compilation failure
+[ERROR] /functional-error-handling-in-kotlin/src/main/kotlin/in/rcard/nullable/NullableTypeErrorHandling.kt:[21,26] Only safe (?.) or non-null asserted (!!.) calls are allowed on a nullable receiver of type Job?
+```
+
+The compiler is warning us that we forgot to handle the case where the list is `null`. As we can see, the compiler is helping us to avoid errors. Let's fix the code:
+
+```kotlin
+fun retrieveSalary(id: JobId): Double =
+ jobs.findById(id)?.salary?.value ?: 0.0
+```
+
+Here, we used the `?.` operator to call a method on a nullable object only if it's not `null`. We must use the `?.` operator for every method call if we have a chain of calls. Finally, we use the "Elvis" operator, `?:`, as a fallback value, in case the job is `null`.
+
+At first, using a nullable value seems less composable than using, for example, the Java `Optional` type. This last type has a lot of functions, `map`, `flatMap`, or `filter`, which make it easy to compose and chain operations.
+
+Kotlin nullable types have nothing to envy to the Java `Optional` type. In fact, the Kotlin standard library provides a lot of functions to handle nullable types. For example, **the `Optional.map` function is equivalent to the `let` scoping function**:
+
+```kotlin
+// Kotlin SDK
+public inline fun T.let(block: (T) -> R): R {
+ contract {
+ callsInPlace(block, InvocationKind.EXACTLY_ONCE)
+ }
+ return block(this)
+}
+```
+
+The `let` function takes a lambda function as input, applying it to the receiver type, and returning the result, as the `map` function (ed., [contracts](https://kotlinlang.org/docs/whatsnew13.html#contracts) is a powerful tool of the language to give the compiler some hints).
+
+To build an example, let's say that the salary of our jobs is in USD, and we want to convert it to EUR. We need a new service to do that:
+
+```kotlin
+class CurrencyConverter {
+ fun convertUsdToEur(amount: Double): Double = amount * 0.91
+}
+```
+
+We can pass the new service to the `JobsService` and use it to get the salary in EUR for a job:
+
+```kotlin
+class JobsService(private val jobs: Jobs, private val converter: CurrencyConverter) {
+
+ // Omissis...
+ fun retrieveSalaryInEur(id: JobId): Double =
+ jobs.findById(id)?.let { converter.convertUsdToEur(it.salary.value) } ?: 0.0
+}
+```
+
+We can easily map a nullable value by mixing up the `let` function with the `?.` operator. Similarly, we can simulate the `filter` function on a nullable value. We'll use the `?.` operator and the `takeIf` function in this case. So, let's add a new function to our service that returns if a job is an Apple job:
+
+```kotlin
+fun isAppleJob(id: JobId): Boolean =
+ jobs.findById(id)?.takeIf { it.company.name == "Apple" } != null
+```
+
+As we can see, we used the `takeIf` in association with the `?.` operator to filter the nullable value. The `takeIf` is an extension function that receives a lambda predicate as a parameter and applies the lambda to the receiver type (not the nullable type):
+
+```kotlin
+// Kotlin SDK
+@OptIn(ExperimentalContracts::class)
+public inline fun T.takeIf(predicate: (T) -> Boolean): T? {
+ contract {
+ callsInPlace(predicate, InvocationKind.EXACTLY_ONCE)
+ }
+ return if (predicate(this)) this else null
+}
+```
+
+Now, we can change the `main` method to use the new function:
+
+```kotlin
+fun main() {
+ val jobs: Jobs = LiveJobs()
+ val currencyConverter = CurrencyConverter()
+ val jobsService = JobsService(jobs, currencyConverter)
+ val appleJobId = JobId(1)
+ val isAppleJob = jobsService.isAppleJob(appleJobId)
+ println("Q: Is the job with id $appleJobId an Apple job?\nA: $isAppleJob")
+}
+```
+If we run the program, we get the expected output:
+
+```text
+Q: Is the job with id JobId(value=1) an Apple job?
+A: false
+```
+
+Although the `?.` operator and the `let` function are extremely powerful, ending with a code with many nested calls is pretty straightforward. For example, let's create a function in our service that returns the sum of the salaries of two jobs:
+
+```kotlin
+fun sumSalaries(jobId1: JobId, jobId2: JobId): Double? {
+ val maybeJob1: Job? = jobs.findById(jobId1)
+ val maybeJob2: Job? = jobs.findById(jobId2)
+ return maybeJob1?.let { job1 ->
+ maybeJob2?.let { job2 ->
+ job1.salary.value + job2.salary.value
+ }
+ }
+}
+```
+
+To overcome the above problem, we can use some sweet functionalities provided by the Kotlin Arrow library. We already imported the dependency from Arrow in the setup section, so we can use it in our code.
+
+The Arrow library provides a lot of functional programming constructs. In detail, for error handling, **it provides a uniform form of monadic list-comprehension** (in Scala, it's called _for-comprehension_) that allows us to write functional code more imperatively and declaratively and avoid the nested calls.
+
+The Arrow DSL applies the same syntax for all the types we can use to handle errors, and it's based on the concept of continuations.
+
+For nullable type, Arrow offers the `nullable` DSL. We can access helpful functions inside the DSL, such as the `ensureNotNull` function and the `bind` extension function. Let's rewrite the `sumSalaries` function using the `nullable` DSL, adding a few logs that we'll use to understand what's going on:
+
+```kotlin
+import arrow.core.raise.*
+
+fun sumSalaries2(jobId1: JobId, jobId2: JobId): Double? = nullable {
+ println("Searching for the job with id $jobId1")
+ val job1: Job = jobs.findById(jobId1).bind()
+ println("Job found: $job1")
+ println("Searching for the job with id $jobId2")
+ val job2: Job = ensureNotNull(jobs.findById(jobId2))
+ println("Job found: $job2")
+ job1.salary.value + job2.salary.value
+}
+```
+
+As we can see, the two functions extract the value from the nullable type. If the value is `null`, then the `nullable` block returns `null` immediately. In the version 2.0 of Arrow, the `nullable` DSL manages both normal and suspend functions inside.
+
+We can give the function a job id that doesn't exist, and then we can check its behavior:
+
+```kotlin
+fun main() {
+ val jobs: Jobs = LiveJobs()
+ val currencyConverter = CurrencyConverter()
+ val jobsService = JobsService(jobs, currencyConverter)
+ val salarySum = jobsService.sumSalaries2(JobId(42), JobId(2)) ?: 0.0
+ println("The sum of the salaries using 'sumSalaries' is $salarySum")
+}
+```
+
+The output of the program is the following:
+
+```text
+Searching for the job with id JobId(value=42)
+The sum of the salaries using 'sumSalaries' is 0.0
+```
+
+As we might expect, the function returns `null` immediately after searching for the `JobId(42)` returns `null`.
+
+Although nullable types offer a reasonable degree of compositionality and full support by the Kotlin language, there are some cases when you can't use it to handle errors. There are some domains where the `null` value is valid, and we can't use it to represent an error. Other times, we need to work with some external libraries that don't support nullable types, such as RxJava or the project Reactor.
+
+Fortunately, Kotlin and the Arrow library provide a lot of alternatives to handle errors functionally. Let's start with the `Option` type.
+
+## 4. Handling Errors with `Option`
+
+Unlike Java, Kotlin doesn't provide a type for handling optional values. As we saw in the previous section, **Kotlin creators preferred introducing nullable values instead of having an `Option` type**.
+
+We can add optional types to the language using the `Arrow` library, which provides a lot of functional programming constructs, including an `Option` type.
+
+The type defined by the Arrow library to manage optional values is the `arrow.core.Option` type. Basically, it's a [Algebraic Data Type (ADT)](https://blog.rockthejvm.com/algebraic-data-types/), technically a sum type, which can be either `Some` or `None`.
+
+The `Some ` type represents a value of type `A`, while the `None` type represents the absence of any value. In other words, `Option` is a type that can either contain a value or not.
+
+In Arrow, we can create an `Option` value using the available constructors directly:
+
+```kotlin
+val awsJob: Some =
+ Some(
+ Job(
+ JobId(1),
+ Company("AWS"),
+ Role("Software Engineer"),
+ Salary(100_000.00),
+ ),
+ )
+val noJob: None = None
+```
+
+The library also provides some helpful extension functions to create `Option` values:
+
+```kotlin
+val appleJob: Option =
+ Job(
+ JobId(2),
+ Company("Apple, Inc."),
+ Role("Software Engineer"),
+ Salary(70_000.00),
+ ).some()
+val noAppleJob: Option = none()
+```
+
+Be careful: Invoking the `some()` function on a `null` value will return a `Some(null)`. If you want to create an `Option` value from a nullable value, you should use the `Option.fromNullable` function:
+
+```kotlin
+val microsoftJob: Job? =
+ Job(
+ JobId(3),
+ Company("Microsoft"),
+ Role("Software Engineer"),
+ Salary(80_000.00)
+ )
+val maybeMsJob: Option = Option.fromNullable(microsoftJob)
+val noMsJob: Option = Option.fromNullable(null) // noMsJob is None
+```
+
+Instead, to convert a nullable value to an `Option` value, we can also use the `toOption()` extension function:
+
+```kotlin
+val googleJob: Option =
+ Job(
+ JobId(4),
+ Company("Google"),
+ Role("Software Engineer"),
+ Salary(90_000.00),
+ ).toOption()
+val noGoogleJob: Option = null.toOption() // noGoogleJob is None
+```
+
+Now that we know how to create `Option` values let's see how to use it. First of all, we make the version of the `Jobs` module that uses the `Option` type:
+
+```kotlin
+interface Jobs {
+
+ fun findById(id: JobId): Option
+}
+
+class LiveJobs : Jobs {
+
+ override fun findById(id: JobId): Option = try {
+ JOBS_DATABASE[id].toOption()
+ } catch (e: Exception) {
+ none()
+ }
+}
+```
+
+Again, we're not interested in the cause of the error. We want to handle it. If the `findById` function fails, we return a `None` value. As you may guess, **the `Option` type is a sealed class**, so we can use the `when` expression to handle the two possible cases. We can define a function that prints the job information of a job id if it exists:
+
+```kotlin
+class JobsService(private val jobs: Jobs) {
+
+ fun printOptionJob(jobId: JobId) {
+ val maybeJob: Option = jobs.findById(jobId)
+ when (maybeJob) {
+ is Some -> println("Job found: ${maybeJob.value}")
+ is None -> println("Job not found for id $jobId")
+ }
+ }
+}
+```
+
+In this case, we can call the `value` property of the `Some` type to get the actual value because Kotlin is smart casting the original `maybeJob` value to the `Some` type.
+
+If we call the above function with the id of a job present in the database, aka `JobId(1)`, we get the expected output:
+
+```text
+Job found: Job(id=JobId(value=2), company=Company(name=Apple, Inc.), role=Role(name=Software Engineer), salary=Salary(value=70000.0))
+```
+
+However, if we use a job id that is not associated with any job, we get the following output:
+
+```text
+Job not found for id JobId(value=42)
+```
+
+However, working with pattern matching is only sometimes very convenient. A lot of time, we need to transform and combine different `Option` values. As in Scala, the `Option` type is a [monad](https://blog.rockthejvm.com/monads/), so we can use the `map` and `flatMap` functions to transform and combine `Option` values. Let's see an example.
+
+Imagine we want to create a function that returns the gap between the job salary given a job id and the maximum salary for the same company. We want to return `None` if the job doesn't exist. To implement such a function, first, we need to add a `findAll` method to our `Jobs` interface:
+
+```kotlin
+interface Jobs {
+
+ // Omissis...
+ fun findAll(): List
+}
+
+class LiveJobs : Jobs {
+
+ // Omissis...
+ override fun findAll(): List = JOBS_DATABASE.values.toList()
+}
+```
+
+Then, we can implement the first version of the function calculating the salary gap using direct `map` and `flatMap` functions:
+
+```kotlin
+class JobsService(private val jobs: Jobs) {
+
+ fun getSalaryGapWithMax(jobId: JobId): Option {
+ val maybeJob: Option = jobs.findById(jobId)
+ val maybeMaxSalary: Option =
+ jobs.findAll().maxBy { it.salary.value }.toOption().map { it.salary }
+ return maybeJob.flatMap { job ->
+ maybeMaxSalary.map { maxSalary ->
+ maxSalary.value - job.salary.value
+ }
+ }
+ }
+}
+```
+
+We can check that everything is working by invoking the `getSalaryGapWithMax` in our `main` function:
+
+```kotlin
+fun main() {
+ val jobs: Jobs = LiveJobs()
+ val jobsService = JobsService(jobs)
+ val appleJobId = JobId(1)
+ val salaryGap: Option = jobsService.getSalaryGapWithMax(appleJobId)
+ println("The salary gap between $appleJobId and the max salary is ${salaryGap.getOrElse { 0.0 }}")
+}
+```
+
+If we run the above code, we get the expected following output:
+
+```text
+The salary gap between JobId(value=1) and the max salary is 20000.0
+```
+
+As we said, **the Kotlin language does not support the _for-comprehension_ syntax**, so the sequences of nested calls to the `flatMap` and `map` function repeatedly can be a bit confusing. Again, the Arrow library gives us help and defines an `option` DSL to call functions on `Option` values in a more readable way.
+
+The `option` DSL is similar to the `nullable` DSL, but works on the `Option` type instead. As for the `nullable` counterpart, we have two flavors of the DSL. The one accepting a suspendable lambda with receiver is `option`, and the one taking a non-suspendable lambda with receiver is called `option.eager`. The receiver of the former DSL is an `arrow.core.continuations.OptionEffectScope`. In contrast, the latter DSL accepts an `arrow.core.continuations.OptionEagerEffectScope`:
+
+```kotlin
+// Arrow SDK
+public object option {
+ public inline fun eager(crossinline f: suspend OptionEagerEffectScope.() -> A): Option =
+ // Omissis...
+
+ public suspend inline operator fun invoke(crossinline f: suspend OptionEffectScope.() -> A): Option =
+ // Omissis...
+}
+```
+
+Let's change the above function to use the `option` DSL:
+
+```kotlin
+fun getSalaryGapWithMax2(jobId: JobId): Option = option.eager {
+ println("Searching for the job with id $jobId")
+ val job: Job = jobs.findById(jobId).bind()
+ println("Job found: $job")
+ println("Searching for the job with the max salary")
+ val maxSalaryJob: Job = jobs.findAll().maxBy { it.salary.value }.toOption().bind()
+ println("Job found: $maxSalaryJob")
+ maxSalaryJob.salary.value - job.salary.value
+}
+```
+
+Again, inside the DSL, the `bind` function is available. If you remember from the previous section, the `bind` function is defined as a member extension function of the `EagerEffectScope` on the `Option` type. It extracts the value from the `Option` if it is a `Some` value. Otherwise, it eagerly returns `None` to the whole DSL:
+
+```kotlin
+// Arrow SDK
+public value class OptionEagerEffectScope(private val cont: EagerEffectScope) : EagerEffectScope {
+
+ // Omissis...
+ public suspend fun Option.bind(): B = bind { None }
+}
+
+public suspend fun Option.bind(shift: () -> R): B =
+ when (this) {
+ None -> shift(shift())
+ is Some -> value
+ }
+```
+
+Let's check it out. We change the `main` function to call the `getSalaryGapWithMax2` function, passing a not existing job id:
+
+```kotlin
+fun main() {
+ val jobs: Jobs = LiveJobs()
+ val jobsService = JobsService(jobs)
+ val salarySum = jobsService.getSalaryGapWithMax2(JobId(42))
+ println("The sum of the salaries using 'sumSalaries' is ${salarySum.getOrElse { 0.0 }}")
+}
+```
+
+We can check that the function immediately returns the `None` value without executing the rest of the code:
+
+```text
+Searching for the job with id JobId(value=42)
+The sum of the salaries using 'sumSalaries' is 0.0
+```
+
+In addition, the `option` DSL also integrates with nullable types through the `ensureNotNull` function. In this case, if present, the function extracts the value from a nullable type. Otherwise, it collapses the execution of the whole lambda in input to the `option` DSL, returning the `None` value. Then, we can rewrite the above example as follows:
+
+```kotlin
+fun getSalaryGapWithMax3(jobId: JobId): Option = option.eager {
+ val job: Job = jobs.findById(jobId).bind()
+ val maxSalaryJob: Job = ensureNotNull(
+ jobs.findAll().maxBy { it.salary.value },
+ )
+ maxSalaryJob.salary.value - job.salary.value
+}
+```
+
+Last but not least, the `nullable` DSL we've seen in the previous section integrates smoothly with the `Option` type. In this case, the `bind` function called on a `None` type will eagerly end the whole block returning a `null` value:
+
+```kotlin
+fun getSalaryGapWithMax4(jobId: JobId): Double? = nullable {
+ println("Searching for the job with id $jobId")
+ val job: Job = jobs.findById(jobId).bind()
+ println("Job found: $job")
+ println("Searching for the job with the max salary")
+ val maxSalaryJob: Job = ensureNotNull(
+ jobs.findAll().maxBy { it.salary.value },
+ )
+ println("Job found: $maxSalaryJob")
+ maxSalaryJob.salary.value - job.salary.value
+}
+```
+
+We can quickly test it by giving the `getSalaryGapWithMax4` function a job id that is not present in the database and checking logs:
+
+```kotlin
+fun main() {
+ val jobs: Jobs = LiveJobs()
+ val jobsService = JobsService(jobs)
+ val fakeJobId = JobId(42)
+ val salaryGap: Double? = jobsService.getSalaryGapWithMax4(fakeJobId)
+ println("The salary gap between $fakeJobId and the max salary is ${salaryGap ?: 0.0}")
+}
+```
+
+The logs we get are the following, highlighting the fact that the `bind` function is called on a `None` value, and the whole block is immediately ended:
+
+```text
+Searching for the job with id JobId(value=42)
+The salary gap between JobId(value=42) and the max salary is 0.0
+```
+
+## 5. Conclusions
+
+This article introduced the meaning of functional error handling in Kotlin. We started showing why we shouldn't rely on exceptions to handle errors. Then, we introduced two strategies to handling errors that forget the cause of errors: Kotlin nullable types and the Arrow `Option` type. Moreover, we saw how the Arrow library provides useful DSL to work with both nullable types and the `Option` type.
+
+In the next part of this series, we will see different strategies that allow us to propagate the cause of errors, such as the Kotlin `Result` type and the Arrow `Either` type.
+
+If you found this article too difficult, you can quickly get the experience you need by following the complete [Kotlin Essentials course](https://rockthejvm.com/p/kotlin-essentials) on Rock the JVM.
+
+## 6. Appendix: Maven Configuration
+
+We give you the maven configuration we used during the examples. Here is the `pom.xml` file:
+
+```xml
+
+
+ 4.0.0
+
+ in.rcard
+ functional-error-handling-in-kotlin
+ 0.0.1-SNAPSHOT
+
+
+ 1.8.20
+ 1.1.5
+
+
+
+
+ org.jetbrains.kotlin
+ kotlin-stdlib
+ ${kotlin.version}
+
+
+ io.arrow-kt
+ arrow-core
+ ${arrow-core.version}
+ pom
+
+
+
+
+ ${project.basedir}/src/main/kotlin
+ ${project.basedir}/src/test/kotlin
+
+
+
+ org.jetbrains.kotlin
+ kotlin-maven-plugin
+ ${kotlin.version}
+
+
+
+ compile
+
+ compile
+
+
+
+
+ test-compile
+
+ test-compile
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+ 7
+ 7
+
+
+
+
+
+```
diff --git a/_posts/2023-05-23-best-scala-courses.md b/_posts/2023-05-23-best-scala-courses.md
new file mode 100644
index 000000000000..8b225121660c
--- /dev/null
+++ b/_posts/2023-05-23-best-scala-courses.md
@@ -0,0 +1,175 @@
+---
+title: "A Comprehensive Guide to Choosing the Best Scala Course"
+date: 2023-05-23
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [scala, learning]
+excerpt: "Discover the best Scala course for your learning journey. Explore the recommended courses and libraries like Cats Effect, ZIO, and Apache Spark."
+---
+
+This article is all about choosing the right Scala course for your journey. In particular, you will learn:
+
+- what to look for in a course
+- what's the best course for beginners if you're just starting out
+- how you should learn Scala
+- what course to take if you already know the basics of Scala and functional programming
+- how to learn other libraries in the Scala ecosystem
+
+**The TLDR - which Scala course should I take?**
+
+- if you're starting out, take [Scala & FP Essentials](https://rockthejvm.com/p/scala)
+- if you're an intermediate Scala user looking to go advanced, take [Advanced Scala](https://rockthejvm.com/p/advanced-scala)
+- after mastering Scala, check out [Cats](https://rockthejvm.com/p/cats), [Cats Effect](https://rockthejvm.com/p/cats-effect) and the [Typelevel](https://rockthejvm.com/p/typelevel-rite-of-passage) ecosystem to get the full Scala developer experience
+- alternative to Cats Effect: take [ZIO](https://rockthejvm.com/p/zio)
+- alternative 2 to Cats Effect is Akka — I wouldn't recommend spending much time there unless you absolutely need it, in which case I have [a whole bundle](https://rockthejvm.com/p/the-akka-bundle)
+- for learning Spark, take this [Spark course set](https://rockthejvm.com/p/the-spark-bundle)
+
+Read on to understand my approach to learning Scala and how you can quickly get any Scala skills you want.
+
+## 1. Introduction
+
+I've made almost 300 hours of courses so far at Rock the JVM, so it's only natural that every once in a while I get one or more of the following questions:
+
+> - _In what order should I take your courses?_
+> - _How should I get started with Scala? Which course should I take?_
+> - _Do you have any tips to learn Scala quickly?_
+
+I wanted to write this article to answer more than just those questions, and give you my deconstructed approach to learning Scala, which learning materials you should take, in what order, and most importantly, _why_ each piece is important.
+
+## 2. How to Learn Scala as a Beginner
+
+Scala is not necessarily aimed at first-time programmers. It's certainly doable, some schools teach Scala to first-year CS students, heck, even [kids can do it](https://twitter.com/BesseIFunction/status/1648560454975868933) with the right teacher. However, the main strength of Scala is in parallel and concurrent systems, and the main selling point of Scala at the time of writing is effect systems like [Cats Effect](https://typelevel.org/cats-effect) and [ZIO](https://zio.dev/), both of which are quite tough to learn unless you're already a professional developer.
+
+So to get started with Scala, I generally recommend at least 1 year of experience in some other language. Scala is usually taught to Java developers, but Java is not mandatory: Python, C++, C#, JavaScript or some other mainstream language is perfectly fine. Because you know a programming language, you should be familiar with the following concepts:
+
+- variables and variable change
+- control structures like if statements
+- repetitions with loops
+- functions and function calls
+- defining your own data structures
+- simple algorithms
+
+With these foundations in place, Scala will come pretty naturally to you, with a small mindset shift towards functional programming which we've talked about elsewhere here on the blog and on the main Rock the JVM site.
+
+### 2.1. What should I look for in a Scala course?
+
+There are a few things to bear in mind if you're considering taking a Scala course:
+
+1. **Make sure that the course doesn't spend too much time on the syntax.** There is _much_ more to Scala than syntax, and the mindset/"thinking in Scala" aspect is the most important. If you see a course that teaches "if statements" 1/3 into the course, look elsewhere.
+2. **Look for a course that focuses on _functional programming_ in Scala.** All mainstream languages are pretty similar when it comes to imperative programming, i.e. variables, loops and the like. If you manage to see a preview of the course, the code inside should look pretty strange to you, and that is good! If the teacher uses variables and loops and it looks like regular Java or Python code but with a different syntax, then that course will miss the essentials of Scala.
+3. **Look for a deconstructed, progressive learning approach.** For Scala, you should see the following topics, in roughly this order:
+ - values (constants), syntax and control structures e.g. if-expressions
+ - object-oriented programming and how to organize your code; methods, traits, inheritance
+ - functional programming and thinking in terms of expressions, function values and chained computations
+ - collections and processing data in a functional way with higher-order functions
+4. If anyone says "for loop" in the course, run.
+
+### 2.2. What's the best Scala course for beginners?
+
+This [Scala & FP Essentials](https://rockthejvm.com/p/scala) is your introduction to Scala. I've recorded it 3 times by now, and the latest version of this course covers Scala 3 from scratch. You will get everything you need to be able to read, understand and contribute to any beginner-to-intermediate Scala project. Most importantly, you'll get a _mindset shift_ which is by far the most important thing to know when you learn Scala or any functional language.
+
+At this stage of learning Scala, you need to learn:
+
+- how to think your code in expressions instead of instructions
+- how to get used to immutable data and constants, which are important no matter what programming language you end up using
+- how to design your own data structures with classes, objects, inheritance
+- how to write idiomatic, elegant and concise code in Scala
+- how to think your code in terms of functional programming principles
+- how functions-as-first-class-citizens work on top of the JVM
+- how to process collections in a straightforward way
+- how to use pattern matching, one of Scala's most powerful features
+
+The course above will check all the above boxes, and more. It has already taught thousands of Scala developers to write and _think_ Scala.
+
+If you have 2 hours free and want to simply have a taste of Scala instead of the full-blown experience, you can check out the [Scala at Light Speed](https://rockthejvm.com/p/scala-at-light-speed) mini-course, which is completely free, and you'll learn the quick foundations of Scala, including OOP, functional programming essentials, and even touching on some advanced features of Scala, such as given/using combos or advanced types.
+
+## 3. How to Learn Scala as an Intermediate-Advanced User
+
+Once you have a solid foundation in Scala and functional programming, it's time to take your skills to the next level. As an intermediate or advanced user, there are some topics you should focus on to enhance your expertise in Scala.
+
+### 3.1. Intermediate Topics to Learn: Concurrency, Contextual Abstractions, Type System
+
+At this point, you can already write and read just about 75% of existing Scala code. The areas that will get you to 95% are concurrency, contextual abstractions and mastering the type system. Let's take them in turn.
+
+**Concurrency**. Scala's strength lies is its support for concurrent and parallel programming. Because Scala was primarily built for the JVM, you will need to understand the JVM threading model. Java developers are at an advantage here, because they've had more time to get accustomed to the mechanics of the JVM. But after you go through the initial mechanics, and with the FP principles you learned in the Scala essentials course, you are ready to discover Scala Futures, which are a functional way of dealing with asynchronous code. You'll compose Futures like you do with regular data types, and you'll learn about [controllable Futures](/controllable-futures), aka Promises.
+
+**Contextual Abstractions**: This is a set of powerful features in Scala that allow you to define context-specific behavior. You'll be able to [pass arguments automatically](/scala-3-given-using), you'll be able to decorate existing types with [new methods](/scala-3-extension-methods), and you'll learn about implicit conversions. These constructs are at the foundation of popular libraries in the Scala ecosystem, like Cats Effect, ZIO and Akka.
+
+**Type System**: Scala has a sophisticated and expressive type system. To become an advanced Scala developer, you should explore advanced type system features such as higher-kinded types, type projections and variance. Understanding these concepts will allow you to write more general and powerful code, while being type-safe 100% of the time.
+
+### 3.2. What's the Best Scala Course for Advanced Users?
+
+For intermediate users looking to deepen their Scala knowledge and get advanced, this [Advanced Scala and Functional Programming course](https://rockthejvm.com/p/advanced-scala) is what you need. This course covers all the advanced topics we discussed above, with functional programming techniques based on them, such as type classes or monads. It will give the skills needed to tackle complex Scala projects and make the most of the language's advanced features.
+
+## 4. What Libraries to Learn as a Scala Developer
+
+Scala has a rich ecosystem of libraries, each built on a radically different set of ideas. Familiarizing yourself with these libraries will greatly expand your capabilities as a Scala developer. Here are some notable libraries you should consider learning:
+
+### 4.1. Cats Effect
+
+[Cats Effect](https://typelevel.org/cats-effect) is a functional effect library that provides a general approach to handling side effects in a purely functional way. Besides the IO monad which gives you the power to write large-scale concurrent applications quite easily, Cats Effect also generalizes computations with type classes, making your code very clear, although it requires quite a bit of discipline to not get frustrated about compiler errors — which are good! Remember that the compiler is supposed to guard you from bugs.
+
+Cats and Cats Effect have started Typelevel, their own mini-ecosystem of libraries, including [http4s](https://http4s.org), [fs2](https://fs2.io/) and [doobie](https://tpolecat.github.io/doobie/), which are heavily used in production.
+
+I talked at length about writing production apps on the Typelevel libraries at Scalar and ScalaMatsuri 2023; you can find a polished recording [here](https://youtu.be/f7IKyXmcT8w).
+
+> If you want to master the Typelevel ecosystem in my biggest course to date, check out the [Typelevel Rite of Passage](https://rockthejvm.com/p/typelevel-rite-of-passage). We build a full-stack application that I deploy to production, on camera.
+
+If you want to learn Cats Effect, I have a long-form [Cats Effect course for Scala developers](https://rockthejvm.com/p/cats-effect). I also teach pure functional programming with [Cats](https://rockthejvm.com/p/cats) if you'd like that instead.
+
+### 4.2. ZIO
+
+[ZIO](https://zio.dev) is another powerful functional effect library for Scala. It's built on similar principles as Cats Effect, although it diverges in a few fundamental aspects.
+
+First, it doesn't use type classes. The ZIO philosophy avoids capability declaration by implicit type class instances (a form of the [tagless final pattern](/tagless-final)). Instead, the ZIO effect already declares all it can do by just looking at its type (dependencies, errors, values).
+
+Second, it opts for more practicality with layers. With the Typelevel ecosystem, as I argued in my conference talk as well, you need a high amount of discipline to organize your code properly. If you have it or it comes natural to you, you're set; not everyone feels like that, and we crave the removal of mental burdens. ZIO embraces the concept of layers, whereby all dependencies for effects are merely declared, and at the end of the application the ZIO library checks if you've passed all dependencies in one big declaration called `provide`. The library takes care to pass the right instance to the right module, freeing your memory from having to remember which module depends on what.
+
+Third, type safety is guaranteed by embracing variance, one of the powerful features of the Scala type system which we explore in the advanced Scala course.
+
+ZIO is super powerful and has an emerging ecosystem of libraries that interact with databases, HTTP, ES, Cassandra, Kafka, GRPC, even Akka wrappers. If you want to learn ZIO, we have a full-blown [ZIO course here](https://rockthejvm.com/p/zio).
+
+### 4.3. Apache Spark
+
+Apache Spark is a popular distributed computing engine that provides high-performance processing of large-scale datasets. It offers Scala APIs and is widely used for big data processing and analytics. Learning Spark will open up opportunities to work on big data projects and leverage the distributed computing capabilities of Scala.
+
+Spark was the Scala ecosystem crown jewel for a long time, and in many respects, is still one of the most powerful pieces of software ever written in Scala. It almost single-handedly exploded the data engineering field.
+
+Apache Spark is for aspiring data engineers. You don't need advanced Scala features to learn Spark, but if you want to get into data engineering, Spark is a great choice. Don't think of Spark as a glorified SQL on trillions of rows, though. Learning Spark will give you principles that you can apply to any data engineering framework.
+
+If you're interested in learning Spark, I recommend the following Spark courses, in order:
+
+- [Spark Essentials](https://rockthejvm.com/p/spark-essentials) for the fundamentals
+- [Spark Optimization](https://rockthejvm.com/p/spark-optimization) for the rare skills and Spark optimization techniques
+- [Spark Performance Tuning](https://rockthejvm.com/p/spark-performance-tuning) for the cluster and infrastructure optimization of Spark
+- [Spark Streaming](https://rockthejvm.com/p/spark-streaming) for the same Spark principles applied to continuous data
+
+### 4.4. Honorable mentions
+
+[Akka](https://akka.io) is a set of libraries for building highly concurrent, distributed, and fault-tolerant applications. I am personally a big fan of Akka because of scalability potential and maturity, but since Lightbend's OSS license reversal, I can only recommend studying Akka if you need it for your work, in which case I have a whole bundle of [Akka courses](https://rockthejvm.com/p/the-akka-bundle).
+
+[Play](https://playframework.com/) is a powerful framework that allows you to build web applications quickly. I have not covered Play in my courses here at Rock the JVM, and the most recent Play books are 5+ years old, so the only true resource is documentation.
+
+[Finagle](https://twitter.github.io/finagle/) is a set of pragmatic Scala libraries used by Twitter in production. They innovated with a few concepts that are now present in various Scala libraries — if you can believe it, Finagle had the `Future` concept before the Scala standard library, and to this day is more powerful, with support for cancellation and distributed invocations.
+
+## 5. Learning Path for Scala Developers
+
+So, the big question: **what's a good order of topics to learn to become a Scala developer?**
+
+I'd recommend the following learning path:
+
+- Scala foundations
+ - basics
+ - advanced concepts that we discussed earlier in the article
+- one FP library ecosystem, either Typelevel or ZIO — don't ask which because you'll get into endless debates, pick one at random and stick with it
+ - the core library (Cats Effect or ZIO)
+ - an HTTP library (Http4s or ZIO HTTP)
+ - a database/persistence library (Doobie or Quill)
+ - a streaming library (FS2 or ZIO Streams)
+ - a functional abstractions library (Cats or ZIO Prelude)
+
+If you want to be a data engineer, follow the Spark learning path above after learning the basics of Scala.
+
+## 5. Conclusion
+
+This article is a comprehensive guide to what you should learn to master Scala. We discussed some essential topics that every Scala developer needs to know, in what order, and which resources to use to learn Scala quickly. We also explored the most important libraries and tools in the Scala ecosystem, as well as a sequenced learning path to becoming a full-blown Scala developer.
diff --git a/_posts/2023-05-26-scala-http4s-authentication.md b/_posts/2023-05-26-scala-http4s-authentication.md
new file mode 100644
index 000000000000..8ed8d43e1714
--- /dev/null
+++ b/_posts/2023-05-26-scala-http4s-authentication.md
@@ -0,0 +1,1079 @@
+---
+title: "HTTP Authentication with Scala and Http4s"
+date: 2023-06-06
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: []
+excerpt: "Learn how you can implement HTTP authentication with Scala and http4s, with 4 different methods: basic, digest, session and JWT tokens."
+toc: true
+toc_label: "In this article"
+---
+
+_by [Herbert Kateu](https://github.com/hkateu)_
+
+> Hey, it's Daniel here. You're reading a big article about the Http4s library.
+> Http4s is one of the most powerful libraries in the Scala ecosystem, and it's part of the Typelevel stack. If you want to master the Typelevel Scala libraries with real-life practice, check out the [Typelevel Rite of Passage](https://rockthejvm.com/p/typelevel-rite-of-passage) course, a full-stack project-based course. It's my biggest and most jam-packed course yet.
+
+For the video version, watch below:
+
+{% include video id="DxZIuvSDvyA" provider="youtube" %}
+
+## 1. Introduction
+
+As a backend developer, authentication is a topic you will encounter numerous times in your career. With numerous authentication methods available, it becomes a challenge not only to choose but also to implement in your application. In this article, we will go through 4 authentication methods and how to implement them in Scala using the Http4s library namely, Basic Authentication, Digest Authentication, Session Authentication, and Token Authentication.
+
+### 1.1 Requirements
+
+To follow along, you will need to add the following to your build.sbt file.
+
+```scala
+val scala3Version = "3.2.2"
+
+val Http4sVersion = "0.23.18"
+val JwtHttp4sVersion = "1.2.0"
+val JwtScalaVersion = "9.3.0"
+
+val http4sDsl = "org.http4s" %% "http4s-dsl" % Http4sVersion
+val emberServer = "org.http4s" %% "http4s-ember-server" % Http4sVersion
+val jwtHttp4s = "dev.profunktor" %% "http4s-jwt-auth" % JwtHttp4sVersion
+val jwtScala = "com.github.jwt-scala" %% "jwt-core" % JwtScalaVersion
+val jwtCirce = "com.github.jwt-scala" %% "jwt-circe" % JwtScalaVersion
+
+lazy val authentication = project
+ .in(file("authentication"))
+ .settings(
+ name := "authentication",
+ version := "0.1.0-SNAPSHOT",
+ scalaVersion := scala3Version,
+ libraryDependencies ++= Seq(
+ emberServer,
+ http4sDsl,
+ jwtHttp4s,
+ jwtScala,
+ jwtCirce
+ )
+ )
+```
+
+The code in this article will be written in Scala 3 but can still be implemented for Scala 2 with very minor changes.
+
+## 2. Authentication
+
+Throughout this article, we will be using the different authentication methods to grant a user access to Http4s `Routes`. Here's a simple implementation of how this will be done.
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.server.*
+import org.http4s.implicits.*
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+
+object SimpleRoutes extends IOApp {
+ val routes: HttpRoutes[IO] =
+ HttpRoutes.of {
+ case GET -> Root / "welcome" / user =>
+ Ok(s"Welcome, ${user}")
+ }
+
+ val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(routes.orNotFound)
+ .build
+
+ override def run(args: List[String]): IO[ExitCode] =
+ server.use(_ => IO.never).as(ExitCode.Success)
+}
+```
+
+In this example, the routes are accessed without authentication. We define our routes using the `HttpRoutes.of` function through which `Http4s` pattern matches against each defined `case`. For our implementation, we have one `case` that accepts `GET` requests from `/welcome/user` where the server responds with `Ok(s"Welcome, ${user}")`. `user` is any alphanumeric value that's passed in the request.
+
+We use `Ember` server through port `8080` to receive requests.
+
+For an in-depth explanation of how to use `Http4s`, there's a good explanation of most concepts [here](/http4s-tutorial/).
+
+Let's test our server.
+
+```bash
+ curl -vv http://localhost:8080/welcome/john
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /welcome/john HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Wed, 24 May 2023 14:45:11 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 13
+<
+* Connection #0 to host localhost left intact
+Welcome, john⏎
+```
+
+When we send a request `http://localhost:8080/welcome/john` to our Rest API, we receive our expected response `Welcome, john`.
+At this point we realize anyone can access this route, to limit access to verified users, the solution is authentication.
+
+## 3. Basic Authentication
+
+Basic authentication is an authentication scheme where user credentials are sent through `HTTP` headers to the server for authentication. The specifications for Basic Authentication are defined in [RFC 7617](https://datatracker.ietf.org/doc/html/rfc7617). Here's how it works.
+
+1. The user's credentials are sent to the server through the use of an `Authorization` header with a `Basic` `scheme`.
+2. `Basic` authentication requires a `username` and `password` to be sent as credentials to the server
+3. The `username` and `password` are joined by a colon `:` and `Base64` encoded before being sent using the `Authorization` header.
+4. The `Authorization` header in the request would be defined as `Authorization:Basic Base64(username:password)`
+5. When the request is received by the server, the credentials are decoded and used to either grant access to a user with a `200 Ok Success` status code or deny access usually with a `401 Unauthorized` status.
+
+### 3.1 Http4s Implementation
+
+Http4s provides an easy way to implement basic authentication using `Middleware` which is simply a function that takes a service and returns another service.
+To begin with we will need a data structure to hold our user information.
+
+```scala
+case class User(id: Long, name: String)
+```
+
+We will also need a function that intercepts the request to validate the user's credentials before granting access to the routes. This is done by use of a `Kleisli`.
+
+```scala
+import cats.data.*
+import org.http4s.Credentials
+import org.http4s.headers.Authorization
+
+val authUserEither: Kleisli[IO, Request[IO], Either[String, User]] = Kleisli { req =>
+ val authHeader: Option[Authorization] = req.headers.get[Authorization]
+ authHeader match {
+ case Some(value) => value match
+ case Authorization(BasicCredentials(creds)) => IO(Right(User(1,creds._1)))
+ case _ => IO(Left("No basic credentials"))
+ case None => IO(Left("Unauthorized"))
+ }
+}
+```
+
+There still a few more steps to make our code work, so keep reading.
+
+We define our Kleisli function `authUserEither` which will take a `Request` and returns an `Either[String,User]`. The function signature `Kleisli[IO, Request[IO], Either[String, User]]` simply translates to `Request[IO] => IO[Either[String, User]]`.
+
+The function checks for the availability of an `Authorization` header which is stored in `authHeader` giving us an `Option[Header]`. Pattern matching on `authHeader` provides us with two cases for the presence or absence of the `Authorization` header.
+
+In case the header is present, we further check for `BasicCredentials` and map those to the `User` case class. The credentials `creds` is a tuple of the form (username,password). At this point in the implementation, one may check against the database to verify the user's credentials, but for simplicity, we return `IO(Right(User(1,creds._1)))` where `creds._1` is the username.
+In case we don't receive something other than an `Authorization` header we return a `Left("No basic credentials")` and in case there's no `Authorization` header, we return a `Left("Unauthorized)`.
+
+The next step is to convert our `Kleisli` to middleware by wrapping it in the `AuthMiddleware` function.
+
+```scala
+val userMiddleware: AuthMiddleware[IO,User] =
+ AuthMiddleware(authUserEither)
+```
+
+To use the middleware, we'll need to modify our routes to the `AuthedRoutes` type instead of `HttpRoutes`
+
+```scala
+val authedRoutes: AuthedRoutes[User,IO] =
+ AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}")
+ }
+```
+
+Previously the `user` value was passed as part of the `URI` in the `GET` request, in this case, the `user` value comes in through the `authUserEither` `Kleisli` function. We will also need Routes to handle failure, which we define next.
+
+```scala
+val onFailure: AuthedRoutes[String, IO] = Kleisli {(_: AuthedRequest[IO,String]) =>
+ OptionT.pure[IO](Response[IO](status = Status.Unauthorized))
+}
+```
+
+The `onFailure` function takes a request and returns a `Response` with `status` code `401 Unauthorized`. Now we can put everything together using the `AuthMiddleware` function to create the middleware. This function takes `authUserEither` and `onFailure` as arguments.
+
+```scala
+val authMiddleware: AuthMiddleware[IO,User] = AuthMiddleware(authUserEither, onFailure)
+```
+
+The final is creating our service by passing our `authedRoutes` to the `authMiddleware` function.
+
+```scala
+val serviceKleisli: HttpRoutes[IO] = authMiddleware(authedRoutes)
+```
+
+With the service now available, we simply pass it to the `Ember` Server and run it.
+
+```scala
+val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(serviceKleisli.orNotFound)
+ .build
+
+override def run(args: List[String]): IO[ExitCode] = server.use(_ => IO.never).as(ExitCode.Success)
+```
+
+Let's test our server
+First, we encode our username and password in Base64 before we put it in our request.
+
+```bash
+$ echo "username:password" | base64
+dXNlcm5hbWU6cGFzc3dvcmQK
+```
+
+The encoded value can now be passed to the `Authorization` header with a `Basic` scheme.
+
+```bash
+$ curl -v -H "Authorization:Basic dXNlcm5hbWU6cGFzc3dvcmQK" http://localhost:8080/welcome
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /welcome HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+> Authorization:Basic dXNlcm5hbWU6cGFzc3dvcmQK
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Sat, 13 May 2023 13:18:09 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 17
+<
+* Connection #0 to host localhost left intact
+Welcome, username⏎
+```
+
+We can also run `curl -vv http://localhost:8080/welcome -u username:password` to give use the same result.
+
+Let's test the failure case.
+
+```bash
+$ curl -v http://localhost:8080/welcome
+
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /welcome HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 401 Unauthorized
+< Date: Sat, 13 May 2023 13:20:14 GMT
+< Connection: keep-alive
+< Content-Length: 0
+<
+* Connection #0 to host localhost left intact
+```
+
+Without the `Authorization` header we get a `401 Unauthorized` status. You can change this response by modifying the `onFailure` function.
+
+Here is the full code:
+
+```scala
+import cats.data.*
+import cats.effect.{IO,IOApp}
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.server.*
+import org.http4s.implicits.*
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+import org.http4s.Credentials
+import org.http4s.headers.Authorization
+
+object BasicExample extends IOApp {
+
+ val authUserEither: Kleisli[IO, Request[IO], Either[String, User]] = Kleisli { req =>
+ val authHeader: Option[Header] = req.headers.get[Authorization]
+ authHeader match {
+ case Some(Authorization(BasicCredentials(creds))) => IO(Right(User(1,creds._1)))
+ case Some(_) => IO(Left("No basic credentials"))
+ case None => IO(Left("Unauthorized"))
+ }
+ }
+
+ val userMiddleware: AuthMiddleware[IO,User] =
+ AuthMiddleware(authUserEither)
+
+ val authedRoutes: AuthedRoutes[User,IO] =
+ AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}")
+ }
+
+ val onFailure: AuthedRoutes[String, IO] = Kleisli { _ =>
+ OptionT.pure[IO](Response[IO](status = Status.Unauthorized))
+ }
+
+ val authMiddleware: AuthMiddleware[IO,User] = AuthMiddleware(authUserEither, onFailure)
+
+ val serviceKleisli: HttpRoutes[IO] = authMiddleware(authedRoutes)
+
+ val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(serviceKleisli.orNotFound)
+ .build
+
+ override def run(args: List[String]): IO[ExitCode] =
+ server.use(_ => IO.never).as(ExitCode.Success)
+}
+```
+
+## 4. Digest Authentication
+
+Similar to Basic Authentication, Digest Authentication also involves passing credentials through the `Authorization` header to the server, however, the Digest Authentication scheme is a two-step process, here's how it works.
+
+1. When a client tries to request information from the server without authorization, the server responds with a `401 Unauthorized` status code, but it also attaches the `realm`, `qop`, and `nounce` to its response.
+ The `realm` is a string supplied by the server which sometimes includes the host name.
+ `qop` stands for the quality of protection, its value might be `auth` (authentication) or 'auth-int' (authentication with integrity)
+ The `nounce` is a unique string generated by the server each time it responds to a `401 status code`.
+
+2. The user at this point provides his or her credentials which are the username and password. These credentials are sent within the `Authorization` header along with some additional information, these include the `realm`, `qop`, and `nounce` which were sent earlier but additionally, the client will attach a `URI`, `cnounce`, `nc` and a `response` to the request.
+
+3. If the credentials checkout, the user will be granted access otherwise access will be denied with the `401 status code`.
+
+I realize there are a lot of new words in the explanation, however, this is beyond the scope of this article, I encourage you to read the [RFC 2617](https://datatracker.ietf.org/doc/html/rfc2617) specification for Digest Authentication for a more in-depth explanation.
+
+### 4.1 Digest Authentication in http4s
+
+Digest Authentication in Http4s is easily implemented using the `DigestAuth` function, but first, we need to define our `User` case class.
+
+```scala
+case class User(id: Long, name: String)
+```
+
+The `DigestAuth` function takes two arguments, the `realm` and a function of type `String => IO[Option[(User, String)]]`. This translates to `"username" => IO[Option(User,"password")]`
+
+Let's define this function.
+
+```scala
+import cats.effect.*
+import org.http4s.server.middleware.authentication.DigestAuth.Md5HashedAuthStore
+
+val ha1: IO[String] = Md5HashedAuthStore.precomputeHash[IO]("username","http://localhost:8080/welcome","password")
+
+val funcPass: String => IO[Option[(User, String)]] = (usr_name: String) =>
+ usr_name match {
+ case "username" => ha1.flatMap(hash => IO(Some(User(1, "username"), hash)))
+ case _ => IO(None)
+ }
+```
+
+The function `funcPass()` receives the username, which it matches to find the correct `User`. The resulting value `IO[Option[User,String]]` is also checked to contain to correct password. It's important to note that the `funcPass()` function can receive either a plaintext or hashed password.
+
+For this example, we decided to use a hashed password. The Http4s digest middleware will check for an `MD5` hashed password in the form `MD5(username:realm:password)` and matches it against the credentials passed in the request. We use the `Md5HashedAuthStore.precomputeHash[IO]()` function to create our hashed value, it takes the `username`, `realm`, and `password` as arguments and returns an `IO[String]`.
+
+```scala
+import org.http4s.server.*
+import org.http4s.server.middleware.authentication.DigestAuth
+
+ val middleware: IO[AuthMiddleware[IO, User]] =
+ DigestAuth.applyF[IO,User]("http://localhost:8080/welcome", Md5HashedAuthStore(funcPass))
+```
+
+To create our `middleware` we use the `.applyF` method on the `DigestAuth` object which takes a `realm` of type `String` and a `store` of type `AuthStore[F, A]`. For our example, we passed the realm as `http://localhost:8080/welcome` and `store` as `Md5HashedAuthStore(funcPass)`.
+
+In case you want to supply a plaintext password, use `PlainTextAuthStore(funcPass)` as your argument, this is imported as follows, `import org.http4s.server.middleware.authentication.DigestAuth.PlainTextAuthStore`.
+
+Your choice of `store` determines how the `username` and `password` will be validated.
+
+Just like Basic Authentication, we'll also need `AuthedRoutes`.
+
+```scala
+import org.http4s.*
+import org.http4s.dsl.io.*
+
+val authedRoutes: AuthedRoutes[User,IO] =
+ AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}")
+ }
+```
+
+To create our service, we call our routes through our `AuthMiddleware[IO, User]()` (middleware) function defined previously.
+
+```scala
+val digestService: IO[HttpRoutes[IO]] =
+ middleware.map(wrapper => wrapper(authedRoutes))
+```
+
+Because `AuthMiddleware[IO, User]` is wrapped in an `IO` we need to call `map` on `middleware` to use it. This gives us a service (`digestService`) of type `HttpRoutes[IO]` also wrapped in an `IO`.
+
+Finally, we pass our service to the Ember server and run it.
+
+```scala
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+
+def server(service: IO[HttpRoutes[IO]]): IO[Resource[cats.effect.IO, Server]] =
+ service.map { svc =>
+ EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(svc.orNotFound)
+ .build
+ }
+
+ override def run(args: List[String]): IO[ExitCode] = server(digestService).flatMap(s => s.use(_ => IO.never)).as(ExitCode.Success)
+```
+
+Here we define a function `server()` which pulls our service out of the `IO` and passes it to the `EmberServiceBuilder`'s `.withHttpApp()` method. This returns our server `Resource` of type `IO[Resource[IO, Server]]` also wrapped in an `IO`.
+
+To run our server we need to call `flatMap` on `server(digestService)` then call `.use(_ => IO.never)` on `Resource[cats.effect.IO, Server]`, and finally end with `ExitCode.Success`.
+
+We can now test our server.
+
+```bash
+curl -vv http://localhost:8080/welcome --digest -u username:password
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+* Server auth using Digest with user 'username'
+> GET /welcome HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 401 Unauthorized
+< Date: Wed, 31 May 2023 10:36:45 GMT
+< Connection: keep-alive
+< WWW-Authenticate: Digest realm="http://localhost:8080/welcome",qop="auth",nonce="296df178c6088b7dbd962177f58270f74475e42b"
+< Content-Length: 0
+<
+* Connection #0 to host localhost left intact
+* Issue another request to this URL: 'http://localhost:8080/welcome'
+* Found bundle for host localhost: 0x557f7bfc1650 [serially]
+* Re-using existing connection! (#0) with host localhost
+* Connected to localhost (::1) port 8080 (#0)
+* Server auth using Digest with user 'username'
+> GET /welcome HTTP/1.1
+> Host: localhost:8080
+> Authorization: Digest username="username", realm="http://localhost:8080/welcome", nonce="296df178c6088b7dbd962177f58270f74475e42b", uri="/welcome", cnonce="MzFmMGVkNzAzZTU2YmNmOGEwODcwZjdhYTM0NjBlYjI=", nc=00000001, qop=auth, response="4b73fd6215bd05b30fc8bcd845ef449c"
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Wed, 31 May 2023 10:36:45 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 17
+<
+* Connection #0 to host localhost left intact
+Welcome, username⏎
+```
+
+`curl` provides a `--digest` flag which takes the username and password and does the initial request followed by a second request with credentials attached in one step.
+
+If the wrong password or username is provided, the server will respond with a `401 Unauthorized`, its also important to note that if the Authorization header is missing or the scheme is anything but `Digest`, the request will fail with a `401 Unauthorized` as well.
+
+Here's the full code:
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.server.*
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+import org.http4s.server.middleware.authentication.DigestAuth
+import org.http4s.server.middleware.authentication.DigestAuth.Md5HashedAuthStore
+
+object DigestExample extends IOApp {
+ case class User(id: Long, name: String)
+
+ val ha1: IO[String] = Md5HashedAuthStore.precomputeHash[IO]("username","http://localhost:8080/welcome","password")
+ val funcPass: String => IO[Option[(User, String)]] = (usr_name: String) =>
+ usr_name match {
+ case "username" => ha1.flatMap(hash => IO(Some(User(1, "username"), hash)))
+ case _ => IO(None)
+ }
+
+ val middleware: IO[AuthMiddleware[IO, User]] =
+ DigestAuth.applyF[IO,User]("http://localhost:8080/welcome", Md5HashedAuthStore(funcPass))
+
+ val authedRoutes: AuthedRoutes[User,IO] =
+ AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}")
+ }
+
+ val digestService: IO[HttpRoutes[IO]] =
+ middleware.map(wrapper => wrapper(authedRoutes))
+
+ def server(service: IO[HttpRoutes[IO]]): IO[Resource[IO, Server]] =
+ service.map { svc =>
+ EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(svc.orNotFound)
+ .build
+ }
+
+ override def run(args: List[String]): IO[ExitCode] =
+ server(digestService).flatMap(s => s.use(_ => IO.never)).as(ExitCode.Success)
+}
+```
+
+## 5. Session Authentication
+
+Session authentication is an authentication technique where the server keeps track of session information after the user has logged in. It works in the following way.
+
+1. When a user logs into a website after verification of username and password, the server responds with a generated unique string which is passed in the `Response` and set as a cookie on the client's side.
+2. This unique string is always passed whenever there are subsequent requests made by the client. If the unique string is valid, access is granted otherwise access is denied.
+3. This cookie can also be valid for a specific amount of time, after which it expires and is automatically removed on the client side.
+4. If the client logs out, the server will remove the cookie from the client side as well.
+
+### 5.1 Session Authentication in Http4s
+
+For this implementation, we are going to reuse our code from the Digest Authentication section. We're going to illustrate this technique with an easy cipher — which should otherwise not be used in production — for you to understand the mechanics.
+
+The first thing we need is a way to generate the unique code that we will send to the client.
+
+```scala
+import java.time.LocalDateTime
+import java.util.Base64
+import java.nio.charset.StandardCharsets
+
+val today: String = LocalDateTime.now().toString()
+def setToken(user: String, date: String):String =
+ Base64.getEncoder.encodeToString(s"${user}:{$today}".getBytes(StandardCharsets.UTF_8))
+```
+
+> NOTE: do not use this exact encoding in production!
+> This example is for illustration to ease understanding. Base64 is easily deciphered, and an attacker can easily impersonate someone by knowing their username and a rough time when the server was started.
+> For true security, such tokens should be signed by a private key. Look [here](https://http4s.org/v1/docs/auth.html#cookies) for more info.
+
+Here we create the unique code or token using the `setToken` function which takes a username as `user` and a date computed by `LocalDateTime.now().toString()`, these values are then combined and encoded using Java's `Base64` utility.
+
+Next, we attach a new cookie to the response, this happens after a successful login.
+
+We'll also define a function that will decode the token when it's received from the client.
+
+```scala
+import scala.util.*
+
+def getUser(token: String): Try[String] = Try((new String(Base64.getDecoder.decode(token))).split(":")(0))
+```
+
+The `getUser` function will take the token and try to decode it, this can either succeed or fail as we will see later.
+
+We modify the `authedRoutes` from our Digest authentication example to add a cookie with the token information
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+
+case class User(id: Long, name: String)
+val authedRoutes: AuthedRoutes[User,IO] =
+ AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}").map(_.addCookie(ResponseCookie("sessioncookie", setToken(user.name, today), maxAge = Some(86400))))
+ }
+```
+
+We set our session token as a cookie on the client side using the `addCookie` function on the `Response` object. The `ResponseCookie` object takes the cookie name, `sessioncookie`, the cookie value which we set using the `setToken` function which we call with the user name and the current datetime, and finally, the maxAge, which is how long the token will persist. This is set to `Some(86400)` equal to 24 hours.
+
+Separate routes are also defined that will be accessed using our session data.
+
+```scala
+val cookieAccessRoutes = HttpRoutes.of[IO] {
+ case GET -> Root / "statement" =>
+ Ok("Financial statement processing...")
+ case GET -> Root / "logout" =>
+ Ok("Logging out...").map(_.removeCookie("sessioncookie"))
+}
+```
+
+The `cookieAccessRoutes` service contains two routes, `/statement` which returns financial statement information, and `/logout` which removes the `sessioncookie` token when the user is logging out. It uses the `removeCookie` function on the `Response` object.
+
+We can now define our session authentication service.
+
+```scala
+import cats.data.*
+import org.http4s.headers.Cookie
+
+def checkSessionCookie(cookie: Cookie):Option[RequestCookie] =
+ cookie.values.toList.find(_.name == "sessioncookie")
+
+def modifyPath(user: String):Path =
+ Uri.Path.fromString(s"/statement/$user")
+
+def cookieCheckerService(service: HttpRoutes[IO]): HttpRoutes[IO] = Kleisli { req =>
+ val authHeader: Option[Cookie] = req.headers.get[Cookie]
+ OptionT.liftF(authHeader.fold(Ok("No cookies")) { cookie =>
+ checkSessionCookie(cookie).fold(Ok("No token")) { token =>
+ getUser(token.content).fold(
+ _ => Ok("Invalid token"),
+ user => service.orNotFound.run(req.withPathInfo(modifyPath(user)))
+ )
+ }
+ })
+}
+```
+
+In http4s one can define custom middleware through the use of a `Kleisli`, here we define a function `cookieCheckerService()` that takes a service of type `HttpRoutes[IO]` and returns another service. This Kleisli contains three helper functions, `checkSessionCookie()`, `getUser()`, and `modifyPath()` which we will break down in the following section.
+
+The `cookieCheckerService()` function will intercept the request and check if it contains any cookies, this is done using the `.get[Cookie]` function on the request headers, which returns an `Option[Cookie]`. When we call `fold` on `authHeader`, we handle the absence of cookies by responding with `Ok("No cookies)` otherwise if cookies are present we check for our `sessioncookie`.
+
+The `checkSessionCookie()` function is then called to check for the presence of the `sessioncookie` within the list of cookies sent through the request, this is done by running `cookie.values.toList.find(_.name == "sessioncookie")` and returns an `Option[RequestCookie]`.
+
+When we call `fold` on the `checkSessionCookie(cookie)` we handle the absence of the `sessioncookie` by returning `Ok("No token")`, otherwise we retrieve the user name by calling `getUser(token.content)`. `token.content` is the token string passed within the `sessioncookie` cookie. `getUser()` returns a Try[String].
+
+If the `getUser()` function fails, we return `Ok("Invalid token")`, otherwise we return a `service` of type `HttpRoutes[IO]` with a modified path. The `.withPathInfo()` method on our request object takes a new `Path` object to which it will forward the request. The `modifyPath()` function takes our user as an argument and returns a new path, `s"/statement/$user"` with the user name inserted. This is done by calling `Uri.Path.fromString(s"/statement/$user")`.
+
+Let's define a router to handle our two services.
+
+```scala
+import org.http4s.server.*
+
+val serviceRouter =
+ Router(
+ "/login" -> digestService,
+ "/" -> cookieCheckerService(cookieAccessRoutes)
+ )
+```
+
+The `/login` path will receive our login request which will be handled by digestService while the sessions will be handled by the `cookieCheckerService()` function taking the `cookieAccessRoutes` service as an argument.
+
+Finally, we can run our server with our new serviceRouter.
+
+```scala
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+
+val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(serviceRouter.orNotFound)
+ .build
+
+override def run(args: List[String]): IO[ExitCode] = server.use(_ => IO.never).as(ExitCode.Success)
+```
+
+Let's test our service, first by logging in.
+
+```bash
+curl -vv http://localhost:8080/login/welcome --digest -u username:password
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+* Server auth using Digest with user 'username'
+> GET /login/welcome HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 401 Unauthorized
+< Date: Mon, 22 May 2023 10:29:53 GMT
+< Connection: keep-alive
+< WWW-Authenticate: Digest realm="http://localhost:8080/welcome",qop="auth",nonce="f6006e52b0212ead5f3e7fa9a600f52ec1667e22"
+< Content-Length: 0
+<
+* Connection #0 to host localhost left intact
+* Issue another request to this URL: 'http://localhost:8080/login/welcome'
+* Found bundle for host localhost: 0x56088ccca650 [serially]
+* Re-using existing connection! (#0) with host localhost
+* Connected to localhost (::1) port 8080 (#0)
+* Server auth using Digest with user 'username'
+> GET /login/welcome HTTP/1.1
+> Host: localhost:8080
+> Authorization: Digest username="username", realm="http://localhost:8080/welcome", nonce="f6006e52b0212ead5f3e7fa9a600f52ec1667e22", uri="/login/welcome", cnonce="MTVkM2RjMGZjZDdhMzlhNTViYjMzMjcyNjE4NTI1MDQ=", nc=00000001, qop=auth, response="b344f127ddf856f962e51ed5030f30ab"
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Mon, 22 May 2023 10:29:54 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 17
+< Set-Cookie: sessioncookie=dXNlcm5hbWU6ezIwMjMtMDUtMjJUMTM6Mjk6MTEuMzc5NTI4fQ==; Max-Age=86400
+<
+* Connection #0 to host localhost left intact
+Welcome, username⏎
+```
+
+The sessioncookie has now been sent to the client and can be used for subsequent requests.
+
+```bash
+curl -vv --cookie "sessioncookie=dXNlcm5hbWU6ezIwMjMtMDUtMjJUMTM6Mjk6MTEuMzc5NTI4fQ==" http://localhost:8080/statement
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /statement HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+> Cookie: sessioncookie=dXNlcm5hbWU6ezIwMjMtMDUtMjJUMTM6Mjk6MTEuMzc5NTI4fQ==
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Mon, 22 May 2023 10:33:20 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 56
+<
+* Connection #0 to host localhost left intact
+Welcome back username, Financial statement processing...⏎
+```
+
+Here's the full code.
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.server.*
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+import org.http4s.server.middleware.authentication.DigestAuth
+import org.http4s.headers.Cookie
+import cats.data.*
+import java.time.LocalDateTime
+import java.util.Base64
+import java.nio.charset.StandardCharsets
+import scala.util.*
+
+object SessionAuth extends IOApp {
+ case class User(id: Long, name: String)
+ val today: String = LocalDateTime.now().toString()
+ def setToken(user: String, date: String):String = Base64.getEncoder.encodeToString(s"${user}:{$today}".getBytes(StandardCharsets.UTF_8))
+ def getUser(token: String): Try[String] = Try(new String(Base64.getDecoder.decode(token)).split(":")(0))
+
+ val funcPass: String => IO[Option[(User, String)]] = (user_val: String) =>
+ user_val match {
+ case "username" => IO(Some(User(1,"username"),"password"))
+ case _ => IO(None)
+ }
+
+ val middleware:AuthMiddleware[IO, User] = DigestAuth[IO,User]("http://localhost:8080/welcome", funcPass)
+
+ val authedRoutes: AuthedRoutes[User,IO] =
+ AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}").map(_.addCookie(ResponseCookie("sessioncookie", setToken(user.name, today), maxAge = Some(86400))))
+ }
+
+ val digestService: HttpRoutes[IO] =
+ middleware(authedRoutes)
+
+
+ val cookieAccessRoutes = HttpRoutes.of[IO] {
+ case GET -> Root / "statement" / user =>
+ Ok(s"Welcome back $user, Financial statement processing...")
+ case GET -> Root / "logout" =>
+ Ok("Logging out...").map(_.removeCookie("sessioncookie"))
+ }
+
+ def cookieCheckerService(service: HttpRoutes[IO]): HttpRoutes[IO] = Kleisli{ req =>
+ val authHeader: Option[Cookie] = req.headers.get[Cookie]
+ OptionT.liftF(authHeader match {
+ case Some(cookie) =>
+ cookie.values.toList.find { x =>
+ x.name == "sessioncookie"
+ } match {
+ case Some(token) =>
+ getUser(token.content) match {
+ case Success(user) =>
+ service.orNotFound.run((req.withPathInfo(Uri.Path.fromString(s"/statement/$user"))))
+ case Failure(_) => Ok("Invalid token")
+ }
+ case None => Ok("No token")
+ }
+ case None => Ok("No cookies")
+ })
+ }
+
+
+ val serviceRouter =
+ Router(
+ "/login" -> digestService,
+ "/" -> cookieCheckerService(cookieAccessRoutes)
+ )
+
+ val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(serviceRouter.orNotFound)
+ .build
+
+ override def run(args: List[String]): IO[ExitCode] =
+ server.use(_ => IO.never).as(ExitCode.Success)
+```
+
+## 6. JSON Web Token Authentication
+
+For JWT authentication to work, a payload containing information for authentication is transmitted between the client and service as a JSON object. JSON Web Tokens are digitally signed using a secret or a private/public key pair so that the tokens' integrity can be verified by both the server and the client.
+
+### 6.1 JWT authentication in http4s
+
+We'll be using the `http4s-jwt-auth` library to implement JWT's in Http4s.
+First, let's create our token that we will send to the client once he/she logs in.
+
+```scala
+import pdi.jwt.*
+import java.time.Instant
+
+val claim = JwtClaim(content = """{"user":"John", "level":"basic"}""", expiration =
+ Some(Instant.now.plusSeconds(157784760).getEpochSecond), issuedAt = Some(Instant.now.getEpochSecond))
+
+val key = "secretKey"
+
+val algo = JwtAlgorithm.HS256
+
+val token = JwtCirce.encode(claim, key, algo)
+```
+
+To create our token we use the `encode` function from `JwtCirce`, which takes a `secret key`, the encoding algorithm, `JwtAlgorithm.HS256`, and a `claim` passed as a `JwtClaim` object.
+The `JwtClaim` object contains the payload which is stored as a JSON string, `"""{"user":"John", "level":"basic"}"""`, here we include the `user` name and his access `level`, there also `expiration` and `issuedAt` values which are both of type `Option[Long]`.
+
+Now we can define our login route where we pass our JSON Web Token to the client.
+
+```scala
+import cats.effect.*
+import org.http4s.*
+import org.http4s.dsl.io.*
+
+val loginRoutes: HttpRoutes[IO] =
+ HttpRoutes.of[IO] {
+ case GET -> Root / "login" =>
+ Ok(s"Logged In").map(_.addCookie(ResponseCookie("token", token)))
+ }
+```
+
+In this case, the JSON Web Token is stored as a cookie named `token` on the client side.
+
+When a request is received we will need to handle the JSON stored in the payload.
+
+```scala
+import io.circe.*
+
+case class TokenPayLoad(user: String, level: String)
+
+object TokenPayLoad {
+ given decoder: Decoder[TokenPayLoad] = Decoder.instance { h =>
+ for {
+ user <- h.get[String]("user")
+ level <- h.get[String]("level")
+ } yield TokenPayLoad(user,level)
+ }
+}
+```
+
+We can parse the JSON string with the help of `Circe`, first we define a case class named `TokenPayLoad` which will hold our payload information. The companion object `TokenPayLoad` contains a `given` or `implicit` decoder method that `yields` a `Decoder[TokenPayLoad]` object from the JSON string.
+Now we can define a function to authenticate the JSON Web Token when it's passed by the user.
+
+```scala
+import dev.profunktor.auth.jwt.*
+import io.circe.parser.*
+
+case class AuthUser(id: Long, name: String)
+val database = Map("John" -> AuthUser(123,"JohnDoe"))
+
+val authenticate: JwtToken => JwtClaim => IO[Option[AuthUser]] =
+ (token: JwtToken) =>
+ (claim: JwtClaim)
+ => decode[TokenPayLoad](claim.content) match {
+ case Right(payload) =>
+ IO(database.get(payload.user))
+ case Left(_) => IO(None)
+ }
+```
+
+The `authenticate` function signature is of type `JwtToken => JwtClaim => IO[Option[AuthUser]]`. This function receives a `JwtToken` and transforms it into a `JWTClaim`, the payload represented as `claim.content` is then extracted and passed to our decoder function. This function results in an `Either[circe.error, TokenPayLoad]`, we can then pattern match against this value to check if the user exists in our simple database by calling `database.get(payload.user)` and returning either a `Left` with circe.error or `Right` with the `TokenPayLoad`. The final result will be an `IO[Option[AuthUser]]`.
+
+We can now define the middleware and routes.
+
+```scala
+import dev.profunktor.auth.*
+
+val jwtAuth = JwtAuth.hmac(key, algo)
+val middleware = JwtAuthMiddleware[IO, AuthUser](jwtAuth, authenticate)
+
+val authedRoutes: AuthedRoutes[AuthUser,IO] =
+AuthedRoutes.of {
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}")
+}
+```
+
+The middleware is defined by calling `JwtAuthMiddleware[IO, AuthUser](jwtAuth, authenticate)`. It takes the `authenticate` function and a `JwtAuth` object which holds the `key` and `algo` values.
+
+The signature for `middleware` is `Kleisli[[_] =>> cats.data.OptionT[cats.effect.IO, _], ContextRequest[cats.effect.IO, AuthUser], Response[cats.effect.IO]] => Kleisli[[_] =>> cats.data.OptionT[cats.effect.IO, _], Request[cats.effect.IO], Response[cats.effect.IO]]`. This seems daunting but it's important to remember, middleware in Http4s is a service that returns another service.
+
+In our case, this `Kleisli` receives the request, authenticates our `token` then forwards the request to `authRoutes` which is the second service that matches against the routes and returns a `Response`.
+
+```scala
+import cats.implicits.*
+
+val securedRoutes: HttpRoutes[IO] = middleware(authedRoutes)
+
+val service = loginRoutes <+> securedRoutes
+```
+
+`securedRoutes` is now our authentication service created by calling `middleware(authedRoutes)` and returns an `HttpRoutes[IO]`.
+We then compose `service` using the `<+>` operator. `service` now gives us access to both `loginRoutes` and `securedRoutes`.
+
+Finally, we can run our server.
+
+```scala
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+
+val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(service.orNotFound)
+ .build
+
+override def run(args: List[String]): IO[ExitCode] =
+ server.use(_ => IO.never).as(ExitCode.Success)
+```
+
+Let's test our server.
+
+```bash
+curl -vv http://localhost:8080/login
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /login HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Mon, 22 May 2023 13:12:05 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 9
+< Set-Cookie: token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE4NDI1NDU2NjQsImlhdCI6MTY4NDc2MDkwNCwidXNlciI6IkpvaG4iLCAibGV2ZWwiOiJiYXNpYyJ9.VjkUrL6Ud0SINNWhUV8M_fFi9YgU8zxevcasiosRIKg
+<
+* Connection #0 to host localhost left intact
+Logged In⏎
+```
+
+First, we log into the server and receive the JSON Web Token, then we can be connected to the rest of the site as an authorized user.
+
+```bash
+curl -vv -H "Authorization:Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE4NDI1NDU2NjQsImlhdCI6MTY4NDc2MDkwNCwidXNlciI6IkpvaG4iLCAibGV2ZWwiOiJiYXNpYyJ9.VjkUrL6Ud0SINNWhUV8M_fFi9YgU8zxevcasiosRIKg" http://localhost:8080/welcome
+* Trying ::1:8080...
+* Connected to localhost (::1) port 8080 (#0)
+> GET /welcome HTTP/1.1
+> Host: localhost:8080
+> User-Agent: curl/7.71.1
+> Accept: */*
+> Authorization:Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE4NDI1NDU2NjQsImlhdCI6MTY4NDc2MDkwNCwidXNlciI6IkpvaG4iLCAibGV2ZWwiOiJiYXNpYyJ9.VjkUrL6Ud0SINNWhUV8M_fFi9YgU8zxevcasiosRIKg
+>
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Mon, 22 May 2023 13:13:29 GMT
+< Connection: keep-alive
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 16
+<
+* Connection #0 to host localhost left intact
+Welcome, JohnDoe⏎
+```
+
+We receive the expected welcome message `Welcome, JohnDoe⏎ `, note that the JWT token is passed as a `Bearer` token using the `Authorization` header.
+
+In case we receive a wrong token, the server will respond with a `403 Forbidden` status code and an `Invalid access token` message.
+
+In case we receive a request without the `Authorization:Bearer` header or anything other than this specific header, the server will respond with a `403 Forbidden` status code and a `Bearer token not found` message.
+
+Here's the full code.
+
+```scala
+import cats.effect.{IOApp, IO, ExitCode}
+import org.http4s.*
+import org.http4s.dsl.io.*
+import org.http4s.ember.server.*
+import com.comcast.ip4s.*
+import cats.implicits.*
+import dev.profunktor.auth.*
+import dev.profunktor.auth.jwt.*
+import pdi.jwt.*
+import java.time.Instant
+import io.circe.*
+import io.circe.parser.*
+
+object TokenAuth extends IOApp {
+ case class AuthUser(id: Long, name: String)
+ case class TokenPayLoad(user: String, level: String)
+
+ object TokenPayLoad {
+ given decoder: Decoder[TokenPayLoad] = Decoder.instance { h =>
+ for {
+ user <- h.get[String]("user")
+ level <- h.get[String]("level")
+ } yield TokenPayLoad(user,level)
+ }
+ }
+
+ val claim = JwtClaim(content = """{"user":"John", "level":"basic"}""",expiration =
+ Some(Instant.now.plusSeconds(157784760).getEpochSecond), issuedAt = Some(Instant.now.getEpochSecond))
+
+ val key = "secretKey"
+
+ val algo = JwtAlgorithm.HS256
+
+ val token = JwtCirce.encode(claim, key, algo)
+
+ val database = Map("John" -> AuthUser(123,"JohnDoe"))
+
+ val authenticate: JwtToken => JwtClaim => IO[Option[AuthUser]] =
+ (token: JwtToken) => (claim: JwtClaim) =>
+ decode[TokenPayLoad](claim.content) match {
+ case Right(payload) => IO(database.get(payload.user))
+ case Left(_) => IO(None)
+ }
+
+ val jwtAuth = JwtAuth.hmac(key, algo)
+ val middleware = JwtAuthMiddleware[IO, AuthUser](jwtAuth, authenticate)
+
+ val authedRoutes: AuthedRoutes[AuthUser,IO] =
+ AuthedRoutes.of{
+ case GET -> Root / "welcome" as user =>
+ Ok(s"Welcome, ${user.name}")
+ }
+
+ val loginRoutes: HttpRoutes[IO] =
+ HttpRoutes.of[IO] {
+ case GET -> Root / "login" =>
+ Ok(s"Logged In").map(_.addCookie(ResponseCookie("token", token)))
+ }
+
+ val securedRoutes: HttpRoutes[IO] = middleware(authedRoutes)
+
+ val service = loginRoutes <+> securedRoutes
+
+ val server = EmberServerBuilder
+ .default[IO]
+ .withHost(ipv4"0.0.0.0")
+ .withPort(port"8080")
+ .withHttpApp(service.orNotFound)
+ .build
+
+ override def run(args: List[String]): IO[ExitCode] =
+ server.use(_ => IO.never).as(ExitCode.Success)
+}
+```
+
+## 7. Conclusion
+
+This article taught us four different ways of implementing HTTP authentication in Scala using Http4s, these include Basic, Digest, Session, and Token authentication methods. We discovered common techniques used and the important role middleware plays when carrying our authentication in Http4s. This article mainly focused on implementation, I would encourage you to read more about the pros and cons of each authentication method to get more context for your application needs.
diff --git a/_posts/2023-06-16-functional-error-handling-in-kotlin-part-2.md b/_posts/2023-06-16-functional-error-handling-in-kotlin-part-2.md
new file mode 100644
index 000000000000..5700d3184be3
--- /dev/null
+++ b/_posts/2023-06-16-functional-error-handling-in-kotlin-part-2.md
@@ -0,0 +1,935 @@
+---
+title: "Functional Error Handling in Kotlin, Part 2: Result and Either"
+date: 2023-06-16
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kotlin]
+excerpt: "In this article, we continue our journey through functional error handling by looking at the `Result` and `Either` data types and how to use them."
+toc: true
+toc_label: "In this article"
+---
+
+_By [Riccardo Cardin](https://github.com/rcardin)_
+
+In this series [first part](https://blog.rockthejvm.com/functional-error-handling-in-kotlin/), we introduced some of the available strategies to handle errors in a functional fashion using Kotlin and the Arrow library. In this second part, we'll continue our journey by looking at the `Result` and `Either` data types and how to use them to handle errors in a functional way.
+
+For the project's setup, please refer to the first part of this series, in which we set up Maven and the needed dependencies.
+
+For the video version, watch below:
+
+{% include video id="C0B44WBJJmY" provider="youtube" %}
+
+Without further ado, let's get started!
+
+> This article, as the first part, requires existing Kotlin experience. If you need to get it **fast** and with thousands of lines of code and a project under your belt, you'll love [Kotlin Essentials](https://rockthejvm.com/p/kotlin-essentials). It's a jam-packed course on **everything** you'll ever need to work with Kotlin for any platform (Android, native, backend, anything), including less-known techniques and language tricks that will make your dev life easier. Check it out [here](https://rockthejvm.com/p/kotlin-essentials).
+
+## 1. The Domain
+
+We'll use extensively the domain model we introduced in the last article. We want to create an application that manages a job board. The main types of the domain are:
+
+```kotlin
+data class Job(val id: JobId, val company: Company, val role: Role, val salary: Salary)
+
+@JvmInline
+value class JobId(val value: Long)
+
+@JvmInline
+value class Company(val name: String)
+
+@JvmInline
+value class Role(val name: String)
+
+@JvmInline
+value class Salary(val value: Double) {
+ operator fun compareTo(other: Salary): Int = value.compareTo(other.value)
+}
+```
+
+Moreover, we'll simulate a database of jobs using a `Map`:
+
+```kotlin
+val JOBS_DATABASE: Map = mapOf(
+ JobId(1) to Job(
+ JobId(1),
+ Company("Apple, Inc."),
+ Role("Software Engineer"),
+ Salary(70_000.00),
+ ),
+ JobId(2) to Job(
+ JobId(2),
+ Company("Microsoft"),
+ Role("Software Engineer"),
+ Salary(80_000.00),
+ ),
+ JobId(3) to Job(
+ JobId(3),
+ Company("Google"),
+ Role("Software Engineer"),
+ Salary(90_000.00),
+ ),
+)
+```
+
+A `Jobs` module will handle the integration with the database:
+
+```kotlin
+interface Jobs
+```
+
+Now that we have defined the domain model and the module that will contain the algebra to access it, it's time to start implementing the different approaches to handle errors in a functional way.
+
+## 2. The `Result` Type: Lifting the Try-Catch Approach to a Higher Level
+
+Nullable types and the `Option` type we've seen so far are great for handling errors in a functional way. They don't store the cause of the error. In other words, they don't tell us _why_ the error happened.
+
+We can represent an error using different approaches. The first is reusing the `Throwable` type and all its exception subtypes. The Kotlin programming language has the `Result` type for that since version 1.3, **which models the result of an operation that may succeed with an instance of `A` or may result in an exception**. It's similar to the `Try ` type we've seen in the Scala programming language.
+
+Despite what you may guess, the `Result ` type is not defined as a sealed class. It's a value class without any subclass:
+
+```kotlin
+// Kotlin SDK
+@JvmInline
+public value class Result @PublishedApi internal constructor(
+ @PublishedApi
+ internal val value: Any?
+) : Serializable
+```
+
+If you're asking, the `@PublishedApi` marks the `internal` constructor and the `value` as accessible from public extension inline functions but not from anywhere else. Here, the trick to represent success and failure results as values of the `value` attribute, which has the type `Any?` and not `T?`. In case of success, the `value` attribute will contain the value of type `T`, **while in case of failure, it will contain an instance of the `Result.Failure` class**, defined as follows:
+
+```kotlin
+// Kotlin SDK
+internal class Failure(
+ @JvmField
+ val exception: Throwable
+ ) : Serializable
+```
+
+Now that we introduced a bit of internals let's see how we can create instances of the `Result` type. Kotlin defines two different smart constructors for that:
+
+```kotlin
+val appleJob: Result = Result.success(
+ Job(
+ JobId(2),
+ Company("Apple, Inc."),
+ Role("Software Engineer"),
+ Salary(70_000.00),
+ ),
+)
+
+val notFoundJob: Result = Result.failure(NoSuchElementException("Job not found"))
+```
+
+Unfortunately, we've no fancy extension functions defined on the `Result` type, so we can't write anything similar to `job.toResult()`. _C'est la vie_. It's not that hard to define such an extension function:
+
+```kotlin
+fun T.toResult(): Result =
+ if (this is Throwable) Result.failure(this) else Result.success(this)
+
+val result = 42.toResult()
+```
+
+How can we use it now that we know how to build a `Result`? First of all, let's create the version of our `Jobs` module that uses the `Result` type:
+
+```kotlin
+interface Jobs {
+
+ fun findById(id: JobId): Result
+}
+
+class LiveJobs : Jobs {
+
+ override fun findById(id: JobId): Result = try {
+ Result.success(JOBS_DATABASE[id])
+ } catch (e: Exception) {
+ Result.failure(e)
+ }
+}
+```
+
+We decided to handle only unexpected errors with the `Result` type by catching exceptions, not the case in which the job is not found.
+
+The `findById` function is implemented using the `try-catch` approach. However, using the `Result` type together with a `try-catch` block is so common, that the Kotlin SDK defines a `runCatching` function that does exactly that:
+
+```kotlin
+// Kotlin SDK
+public inline fun runCatching(block: () -> R): Result {
+ return try {
+ Result.success(block())
+ } catch (e: Throwable) {
+ Result.failure(e)
+ }
+}
+```
+
+Using the `runCatching` function, we can rewrite the `findById` function as follows:
+
+```kotlin
+override fun findById(id: JobId): Result = runCatching {
+ JOBS_DATABASE[id]
+}
+```
+
+The `runCatching` function is also defined as an extension function on the `Any` type, so we can use it on any object:
+
+```kotlin
+override fun findById(id: JobId): Result = id.runCatching {
+ JOBS_DATABASE[this]
+}
+```
+
+Once we have a `Result`, we can use it differently. The first is to check if the result is a success or failure. We can use the `isSuccess` and `isFailure` properties. To show them, as we made for other types, we can create our `JobsService` using the new flavor of the `Jobs` module, and develop a simple `printOptionJob` method, which prints different messages depending on the result of the `findById` function:
+
+```kotlin
+class JobService(private val jobs: Jobs) {
+
+ fun printOptionJob(jobId: JobId) {
+ val maybeJob: Result = jobs.findById(jobId)
+ if (maybeJob.isSuccess) {
+ maybeJob.getOrNull()?.apply { println("Job found: $this") } ?: println("Job not found for id $jobId")
+ } else {
+ println("Something went wrong: ${maybeJob.exceptionOrNull()}")
+ }
+ }
+}
+```
+
+The above code also introduces some of the extractor methods defined on the `Result` type:
+
+* `getOrNull` returns the value of the `Result` or `null` otherwise.
+* `getOrThrow` returns the value of the `Result` or the throw of the exception contained in the `Result.Failure` instance.
+* `getOrDefault` returns the value of the `Result` or a given default value if the `Result` is a failure.
+* `getOrElse` returns the value of the `Result`, or the result of a given lambda, `onFailure: (exception: Throwable) -> R` if the `Result` is a failure.
+
+Conversely, the method `exceptionOrNull` returns the exception in the `Result.Failure` instance or `null` otherwise.
+
+If we execute the above method using a valid `JobId`, we'll get the following output:
+
+```text
+Job found: Job(id=JobId(value=1), company=Company(name=Apple, Inc.), role=Role(name=Software Engineer), salary=Salary(value=70000.0))
+```
+
+However, it's far more idiomatic to transform and react to values of a `Result` rather than extracting it. To apply a transformation to the value of a `Result`, we can use the classic `map` function:
+
+```kotlin
+val appleJobSalary: Result = appleJob.map { it.salary }
+```
+
+Instead, we can use the `mapCatching` function if the transformation to apply to the value of the `Result` can throw an exception. Let's reuse our `CurrencyConverter` class from the first part of the series. The `CurrencyConverter` is a class that converts amounts from one currency to another one. In detail, we want to develop a method to convert an amount in USD to EUR. This time, we add validation on the input amount:
+
+```kotlin
+class CurrencyConverter {
+ @Throws(IllegalArgumentException::class)
+ fun convertUsdToEur(amount: Double?): Double =
+ if (amount != null && amount >= 0.0) {
+ amount * 0.91
+ } else {
+ throw IllegalArgumentException("Amount must be positive")
+ }
+}
+```
+
+The converter will throw an exception if the input amount is `null` or negative. Let's think about the converter as an external library that we can't change. We can use the `mapCatching` function to convert the salary of a job from USD to EUR, handling the fact that the conversion can fail, throwing an exception. Let's add the converter as a dependency to our `JobService` class, and let's use it to implement a new method:
+
+```kotlin
+class JobService(private val jobs: Jobs, private val currencyConverter: CurrencyConverter) {
+
+ fun getSalaryInEur(jobId: JobId): Result =
+ jobs.findById(jobId)
+ .map { it?.salary }
+ .mapCatching { currencyConverter.convertUsdToEur(it?.value) }
+}
+```
+
+If we execute the above function using an invalid `JobId`, we'll get a `Result` containing the exception thrown by the `convertUsdToEur` function:
+
+```kotlin
+fun main() {
+ val currencyConverter = CurrencyConverter()
+ val jobs = LiveJobs()
+ val maybeSalary = JobService(jobs, currencyConverter).getSalaryInEur(JobId(42))
+ println(maybeSalary)
+}
+```
+
+The output is the following:
+
+```text
+Failure(java.lang.IllegalArgumentException: Amount must be positive)
+```
+
+If we want to recover from a failure `Result`, we can use the `recover` function. This function takes a lambda that will be executed if the `Result` is a failure. The lambda takes as input the exception contained in the `Result.Failure` instance and returns a new value of the same type as the successful type of the original `Result`:
+
+```kotlin
+// Kotlin SDK
+public inline fun Result.recover(transform: (exception: Throwable) -> R): Result {
+ contract {
+ callsInPlace(transform, InvocationKind.AT_MOST_ONCE)
+ }
+ return when (val exception = exceptionOrNull()) {
+ null -> this
+ else -> Result.success(transform(exception))
+ }
+}
+```
+
+The original `T` type must be a subtype of the new `R` type since we return the initial value if the `Result` succeeds.
+
+In our example, we can use the `recover` function to return a default value if no job is found:
+
+```kotlin
+val maybeSalary: Result = JobService(jobs, currencyConverter).getSalaryInEur(JobId(42))
+val recovered = maybeSalary.recover {
+ when (it) {
+ is IllegalArgumentException -> println("The amount must be positive")
+ else -> println("An error occurred ${it.message}")
+ }
+ 0.0
+}
+println(recovered)
+```
+
+If we execute the above code with the same input as the previous example, we can get a more detailed and more precise output than the previous one:
+
+```text
+The amount must be positive
+Success(0.0)
+```
+
+As we did for the `map` function, we can use the `recoverCatching` variant if the lambda passed to the `recover` function can throw an exception.
+
+To execute some side effect with the value of a `Result`, whether successful or a failure, we can use the reliable methods the SDK gives us, i.e., the functions `onSuccess` and `onFailure`. Both functions return the original `Result` instance, so we can chain them:
+
+```kotlin
+// Kotlin SDK
+public inline fun Result.onSuccess(action: (value: T) -> Unit): Result {
+ contract {
+ callsInPlace(action, InvocationKind.AT_MOST_ONCE)
+ }
+ if (isSuccess) action(value as T)
+ return this
+}
+
+public inline fun Result.onFailure(action: (exception: Throwable) -> Unit): Result {
+ contract {
+ callsInPlace(action, InvocationKind.AT_MOST_ONCE)
+ }
+ exceptionOrNull()?.let { action(it) }
+ return this
+}
+```
+
+We can refactor our primary example to use the `onSuccess` and `onFailure` functions:
+
+```kotlin
+val notFoundJobId = JobId(42)
+val maybeSalary: Result =
+ JobService(jobs, currencyConverter).getSalaryInEur(notFoundJobId)
+maybeSalary.onSuccess {
+ println("The salary of jobId $notFoundJobId is $it")
+}.onFailure {
+ when (it) {
+ is IllegalArgumentException -> println("The amount must be positive")
+ else -> println("An error occurred ${it.message}")
+ }
+}
+```
+
+Clearly, the output of the above code is the following since the `JobId` 42 is not present in our database:
+
+```text
+The amount must be positive
+```
+
+If we want to give both the lambda to apply in case of success and the lambda to apply in case of failure, we can use the `fold` function:
+
+```kotlin
+// Kotlin SDK
+public inline fun Result.fold(
+ onSuccess: (value: T) -> R,
+ onFailure: (exception: Throwable) -> R
+): R {
+ contract {
+ callsInPlace(onSuccess, InvocationKind.AT_MOST_ONCE)
+ callsInPlace(onFailure, InvocationKind.AT_MOST_ONCE)
+ }
+ return when (val exception = exceptionOrNull()) {
+ null -> onSuccess(value as T)
+ else -> onFailure(exception)
+ }
+}
+```
+
+As we can see, the `fold` function is not only used to apply side effects. It can be used to transform a `Result` into another type. In fact, many of the transformations we've seen so far are shorthands for applying the `fold` function in particular cases.
+
+The above example can be rewritten using the `fold` function as follows:
+
+```kotlin
+maybeSalary.fold({
+ println("The salary of jobId $notFoundJobId is $it")
+}, {
+ when (it) {
+ is IllegalArgumentException -> println("The amount must be positive")
+ else -> println("An error occurred ${it.message}")
+ }
+})
+```
+
+## 3. Composing `Result` Instances
+
+As we saw in the previous article, we often deal with types (usually some containers) that are "chainable" or "monadic" in structure (more details on [Another Take at Monads: A Way to Generalize Chained Computations](https://blog.rockthejvm.com/another-take-on-monads/)), so the crucial point is how we can compose and combine them. In the first part of this series, we implemented a function that returns the gap between the job salary given a job id and the maximum compensation for the same company. We called the function `getSalaryGapWithMax`.
+
+We want to refactor the example using the `Result` type. First, we need to add the `findAll` function to the `Jobs` interface and implementation:
+
+```kotlin
+interface Jobs {
+
+ fun findAll(): Result>
+ // Omissis...
+}
+
+class LiveJobs : Jobs {
+
+ override fun findAll(): Result> =
+ Result.success(JOBS_DATABASE.values.toList())
+ // Omissis...
+}
+```
+
+To calculate the gap with the max salary, we need to retrieve a job with a given id and all the jobs available. So, we need to compose two `Result` instances.
+
+The Kotlin SDK doesn't provide any form of `flatMap` like function for the `Result` type. So, how can we compose subsequent computations resulting in a `Result`? **Remember that two of the most essential principle of Kotlin are pragmatism and ergonomics**. If we think about it, we already have all the tools to build some monadic style list comprehension without using `flatMap`.
+
+One of the main properties of monadic list-comprehension is the ability to short-circuit the computation if one of the steps fails. Exceptions are very good at doing short-circuiting, so why not use them? We saw that on a `Result`, we can call the `getOrThrow` function to get the value of a `Result` or throw an exception if the `Result` is a failure. So, we can use the `getOrThrow` function to short-circuit the computation if one of the steps fails. However, we want to avoid handling exceptions through `try-catch` blocks. So, we can use the `runCatching` function to wrap the computation again in a `Result`.
+
+To make the code more readable, first, we define an extension function that returns the maximum salary of a list of `Job`:
+
+```kotlin
+fun List.maxSalary(): Result = runCatching {
+ if (this.isEmpty()) {
+ throw NoSuchElementException("No job present")
+ } else {
+ this.maxBy { it.salary.value }.salary
+ }
+}
+```
+
+If the list is empty, we threw a `NoSuchElementException`, wrapping it together in a `Result`. Then, we can use the new function to implement the `getSalaryGapWithMax` function in the `JobService` as follows:
+
+```kotlin
+class JobService(private val jobs: Jobs, private val currencyConverter: CurrencyConverter) {
+
+ // Omissis...
+ fun getSalaryGapWithMax(jobId: JobId): Result = runCatching {
+ val maybeJob: Job? = jobs.findById(jobId).getOrThrow()
+ val jobSalary = maybeJob?.salary ?: Salary(0.0)
+ val jobList = jobs.findAll().getOrThrow()
+ val maxSalary: Salary = jobList.maxSalary().getOrThrow()
+ maxSalary.value - jobSalary.value
+ }
+}
+```
+
+As we can see, **we can forget about the `Result` type during the composition process with this approach, focusing on the success values**. The rise of exceptions and the use of the `runCatching` function allows us to short-circuit the computation if one of the steps fails.
+
+What about the Arrow library and the `Result` type? As we saw for the nullable types, Arrow offers some exciting extensions. First, Arrow adds the `flatMap` function to the `Result` type. If we are Haskell lovers, we can't live without it, and we can use the `flatMap` to compose subsequent computations resulting in a `Result`. Let's try to rewrite the previous example using the `flatMap` function from the Arrow library:
+
+```kotlin
+import arrow.core.flatMap
+
+class JobService(private val jobs: Jobs, private val currencyConverter: CurrencyConverter) {
+
+ // Omissis...
+ fun getSalaryGapWithMax2(jobId: JobId): Result =
+ jobs.findById(jobId).flatMap { maybeJob ->
+ val jobSalary = maybeJob?.salary ?: Salary(0.0)
+ jobs.findAll().flatMap { jobList ->
+ jobList.maxSalary().map { maxSalary ->
+ maxSalary.value - jobSalary.value
+ }
+ }
+ }
+}
+```
+
+As we said in the previous article, the absence of native support for monadic comprehension in Kotlin makes the code less readable if we use sequences of `flatMap` and `map` invocations. However, as we saw both for nullable types and for the `Option` type, **Arrow gives us nice DSLs to deal with the readability problem**. For the `Result`type, the DSL is called `result`:
+
+```kotlin
+// Arrow SDK
+public object result {
+ public inline fun eager(crossinline f: suspend ResultEagerEffectScope.() -> A): Result
+ // Omissis
+
+ public suspend inline operator fun invoke(crossinline f: suspend ResultEffectScope.() -> A): Result
+ // Omissis
+}
+```
+
+The `suspend` and the `eager` version of the DSL define a scope object as the receiver, respectively `arrow.core.continuations.ResultEffectScope` and `arrow.core.continuations.ResultEagerEffectScope`. As we did in the previous article, we'll use the _eager_ flavor of the DSL.
+
+As for the nullable types and the `Option` type, the `result` DSL gives us the `bind` extension function to unwrap the value of a `Result` and use it. If the `Result` fails, the `bind` function will short-circuit the calculation and the whole block inside the `result` DSL returns the failure. The `bind` function is defined as an extension function inside the `ResultEagerEffectScope`:
+
+```kotlin
+// Arrow SDK
+public value class ResultEagerEffectScope(/* Omissis */) : EagerEffectScope {
+
+ // Omissis
+ public suspend fun Result.bind(): B =
+ fold(::identity) { shift(it) }
+}
+```
+
+The `shift` function short-circuits the computation and returns the failure, terminating the continuation chain. Remember, **Arrow implements all the scopes concerning error handling using a continuation-style approach**.
+
+```kotlin
+fun getSalaryGapWithMax3(jobId: JobId): Result = result.eager {
+ println("Searching for the job with id $jobId")
+ val maybeJob: Job? = jobs.findById(jobId).bind()
+ ensureNotNull(maybeJob) { NoSuchElementException("Job not found") }
+ val jobSalary = maybeJob.salary
+ println("Job found: $maybeJob")
+ println("Getting all the available jobs")
+ val jobList = jobs.findAll().bind()
+ println("Jobs found: $jobList")
+ println("Searching for max salary")
+ val maxSalary: Salary = jobList.maxSalary().bind()
+ println("Max salary found: $maxSalary")
+ maxSalary.value - jobSalary.value
+}
+```
+
+This time, we'll change the example's main workflow. We'll use a `NoSuchElementException` to signal the job with the given `jobId` is not present in the database. We used the `ensureNotNull` function to check and apply if a nullable value is not null and apply. In the case of the scope defined in the `result` DSL, the function short-circuits the execution with a `Result` containing the given exception as a failure.
+
+Let's change the `main` function to use the new `getSalaryGapWithMax3` function:
+
+```kotlin
+fun main() {
+ val currencyConverter = CurrencyConverter()
+ val jobs = LiveJobs()
+ val notFoundJobId = JobId(42)
+ val salaryGap: Result =
+ JobService(jobs, currencyConverter).getSalaryGapWithMax3(notFoundJobId)
+ salaryGap.fold({
+ println("Salary gap for job $notFoundJobId is $it")
+ }, {
+ println("There was an error during execution: $it")
+ })
+}
+```
+
+If we run the previous example giving the `JobId` 42 as input, we'll get the following expected output:
+
+```text
+Searching for the job with id JobId(value=42)
+There was an error during execution: java.util.NoSuchElementException: Job not found
+```
+
+As we can see, the execution was short-circuited when the `ensureNotNull` function returned a `Result` containing the `NoSuchElementException` as a failure. However, the `result` DSL does not equal the `runCatching` function. If we throw an exception inside the `result` DSL, it will be propagated to the caller and bubble up through the call stack. Let's try to throw an exception inside the `result` DSL:
+
+```kotlin
+fun main() {
+ val result: Result = result.eager {
+ throw RuntimeException("Boom!")
+ }
+}
+```
+
+If we run the above code, we `RuntimeException` will escape the `result` DSL, printing the following stack trace:
+
+```text
+Exception in thread "main" java.lang.RuntimeException: Boom!
+ at in.rcard.result.ResultTypeErroHandlingKt$main$$inlined$eager-IoAF18A$1.invokeSuspend(result.kt:43)
+ at in.rcard.result.ResultTypeErroHandlingKt$main$$inlined$eager-IoAF18A$1.invoke(result.kt)
+ at in.rcard.result.ResultTypeErroHandlingKt$main$$inlined$eager-IoAF18A$1.invoke(result.kt)
+ at arrow.core.continuations.DefaultEagerEffect$fold$1.invokeSuspend(EagerEffect.kt:190)
+ at arrow.core.continuations.DefaultEagerEffect$fold$1.invoke(EagerEffect.kt)
+ at arrow.core.continuations.DefaultEagerEffect$fold$1.invoke(EagerEffect.kt)
+ at arrow.core.continuations.DefaultEagerEffect.fold(EagerEffect.kt:192)
+ at arrow.core.continuations.ResultKt.toResult(result.kt:10)
+ at in.rcard.result.ResultTypeErroHandlingKt.main(ResultTypeErroHandling.kt:122)
+ at in.rcard.result.ResultTypeErroHandlingKt.main(ResultTypeErroHandling.kt)
+```
+
+If we want to use some computation that can throw an exception, we can use the `runCatching` function inside the `result` DSL:
+
+```kotlin
+fun main() {
+ result.eager {
+ runCatching {
+ throw RuntimeException("Boom!")
+ }.bind()
+ }
+}
+```
+
+However, sometimes we want to map errors in custom types that don't belong to the `Throwable` hierarchy. For example, we can map a `NoSuchElementException` to a `JobNotFound` type or any rich and meaningful type we want. To do this, we need another strategy to handle errors. It's time to introduce the `Either` type.
+
+## 4. Type-safe Error Handling: The `Either` Type
+
+Let's now introduce the `Either` type for error handling. Kotlin doesn't ship the `Either` type with the standard SDK. We need Arrow to add it to the game. The structure of `Either` is that of an [Algebraic Data Type](https://blog.rockthejvm.com/algebraic-data-types/) (ADT). In detail, it's a sum type that can contain either a value `A` wrapped in the type `Right` or a value `E` wrapped in a type `Left`. It's common to associate `Left` instances with the result of a failed computation and `Right` instances with the result of a successful calculation. The `Either` type is defined as follows:
+
+```kotlin
+// Arrow SDK
+public sealed class Either
+public data class Left constructor(val value: A) : Either()
+public data class Right constructor(val value: B) : Either()
+```
+
+The `Either` type is a sealed class, so it cannot be extended outside the Arrow library, and the compiler can check if all the possible cases are handled in a `when` expression.
+
+Since we can now use any type to represent errors, we can create an ADT on error causes. For example, we can define a `JobError` sealed class and extend it with the `JobNotFound` and `GenericError` classes:
+
+```kotlin
+sealed interface JobError
+data class JobNotFound(val jobId: JobId) : JobError
+data class GenericError(val cause: String) : JobError
+```
+
+Let's see how to create an `Either` instance. The `Left` and `Right` classes have a constructor that takes a single parameter. Here's an example of how to create a `Right` instance:
+
+```kotlin
+val appleJobId = JobId(1)
+val appleJob: Either = Right(JOBS_DATABASE[appleJobId]!!)
+```
+
+Here, we forced the type of the left part of the `Either` to be `JobError`. Notice that the constructor returns an `Either` with the left part defined as `Nothing`. The compiler allows us to do this because the `Nothing` type is a subtype of any other type, and **the `Either` is covariant on the left part since it's defined using the `out` keyword** (we already saw variance in previous articles on Scala, [Variance Positions in Scala, Demystified](https://blog.rockthejvm.com/scala-variance-positions/)).
+
+Now, we can create our `Left` type instance:
+
+```kotlin
+val jobNotFound: Either = Left(JobNotFound(appleJobId))
+```
+
+For those who prefer extension functions, Arrow provides the `left` and `right` functions to create `Left` and `Right` instances:
+
+```kotlin
+val anotherAppleJob = JOBS_DATABASE[appleJobId]!!.right()
+val anotherJobNotFound: Either = JobNotFound(appleJobId).left()
+```
+
+Since Arrow defines the `Either` type as a sealed class, we can use the `when` expression to handle all the possible cases taking advantage of the smart casting. For example, in the following `printSalary`function, we can access the `value` attribute of the `Right` instance without any explicit cast:
+
+```kotlin
+fun printSalary(maybeJob: Either) = when (maybeJob) {
+ is Right -> println("Job salary is ${maybeJob.value.salary}")
+ is Left -> println("No job found")
+}
+```
+
+If we want to extract the contained value from an `Either`, we have some functions. The `getOrNull` function returns a nullable type containing the value if it's a `Right` instance. We can use it if we want to discard the error:
+
+```kotlin
+val appleJobOrNull: Job? = appleJob.getOrNull()
+```
+
+Similarly, we can transform an `Either` instance into an `Option` instance using the `getOrNone` function:
+
+```kotlin
+val maybeAppleJob: Option = appleJob.getOrNone()
+```
+
+Then, the `getOrElse` function lets us extract the value contained in a `Right` instance or a default value if it's a `Left` instance:
+
+```kotlin
+val jobCompany: String = appleJob.map { it.company.name }.getOrElse { "Unknown company" }
+```
+
+The `getOrElse` function takes a lambda with the error as a parameter, so we can use it to react differently to different errors:
+
+```kotlin
+val jobCompany2: String =
+ appleJob.map { it.company.name }.getOrElse { jobError ->
+ when (jobError) {
+ is JobNotFound -> "Job not found"
+ is GenericError -> "Generic error"
+ }
+}
+```
+
+Using typed errors has many advantages:
+
+ 1. We can use the type system to check if all the possible cases are handled.
+ 2. The possible causes of failure are listed directly in the function's signature as the left part of the `Either` type. Understanding the possible causes of failure lets us build better tests and error-handling strategies.
+ 3. Typed errors compose better than exceptions.
+
+To prove the above advantages, as we previously did for the `Result` type, it's time to use the `Either` type in our example. Let's change the `Jobs` module to return an `Either` type instead of a `Result` type:
+
+```kotlin
+interface Jobs {
+
+ fun findById(id: JobId): Either
+}
+```
+
+Here we are using the `JobError` ADT we defined so far. At this point, we can know precisely how the function can fail by looking at the signature. Now, we can implement the `LiveJobs` class:
+
+```kotlin
+class LiveJobs : Jobs {
+
+ override fun findById(id: JobId): Either =
+ try {
+ JOBS_DATABASE[id]?.right() ?: JobNotFound(id).left()
+ } catch (e: Exception) {
+ GenericError(e.message ?: "Unknown error").left()
+ }
+}
+```
+
+As we might expect, we're wrapping the happy path with a `Right` instance and all the available error cases with a `Left` instance. We are treating the absence of a job as a logic error, and we're wrapping it with a `JobNotFound` object using the recommended and idiomatic syntax:
+
+```kotlin
+value?.right() ?: error.left()
+```
+
+On the other hand, we catch all the exceptions and wrap them in a `GenericError` instance. The pattern of catching exceptions and wrapping them in a `Left` instance is so common that Arrow provides the `catch` function to do it for us in the companion object of the `Either` type:
+
+```kotlin
+@JvmStatic
+@JvmName("tryCatch")
+public inline fun catch(f: () -> R): Either =
+ try {
+ f().right()
+ } catch (t: Throwable) {
+ t.nonFatalOrThrow().left()
+ }
+```
+
+The `nonFatalOrThrow` function checks whether the exception should be handled. The fatal exceptions are the following and their subclasses:
+
+* `VirtualMachineError`
+* `ThreadDeath`
+* `InterruptedException`
+* `LinkageError`
+* `ControlThrowable`
+* `CancellationException`
+
+The subtypes of these errors should not be caught. For example, CancellationException should not be caught because it's used by Kotlin to [cancel coroutines](https://blog.rockthejvm.com/kotlin-coroutines-101/#7-cancellation), and catching it can break normal functioning of coroutines.
+
+Then, we can rewrite the `LiveJobs` class using the `catch` function:
+
+```kotlin
+class LiveJobs : Jobs {
+
+ override fun findById(id: JobId): Either = catch {
+ JOBS_DATABASE[id]
+ }.mapLeft { GenericError(it.message ?: "Unknown error") }
+ .flatMap { maybeJob -> maybeJob?.right() ?: JobNotFound(id).left() }
+}
+```
+
+Wait. We introduced a bunch of new functions here. Let's see them in detail. First, **the `Either` type is right-based**, which means that `Right` is assumed to be the default case to operate on. If it is `Left`, operations like `map`, `flatMap`, return the `Left` value unchanged. In this case, we applied the `flatMap` function to check if the retrieved job was null, eventually creating a `Left` value using the pattern we saw a moment ago.
+
+Moreover, we introduced the `mapLeft` function. This function is similar to the `map` function but applies to the `Left` part of the `Either` type. In this case, we're mapping `Throwable` exceptions to a `GenericError` object. The `mapLeft` function is defined as follows:
+
+```kotlin
+// Arrow SDK
+public inline fun mapLeft(f: (A) -> C): Either =
+ fold({ Left(f(it)) }, { Right(it) })
+```
+
+As we can see, the definition of the `mapLeft` function allows us to introduce another essential function, the `fold` function. We already saw this function in the `Result` type, and we can use it to transform both the `Left` and `Right` instances of the `Either` class into a target type. The `fold` function is defined as follows:
+
+```kotlin
+// Arrow SDK
+public inline fun fold(ifLeft: (left: A) -> C, ifRight: (right: B) -> C): C =
+ when (this) {
+ is Right -> ifRight(value)
+ is Left -> ifLeft(value)
+ }
+```
+
+The `fold` function wraps around the `when` expression. It's crucial in the library, and many of the `getOr*` are implemented using it. For example, let's say we want to get the salary of a job, whether it's a `Left` or a `Right` instance. We can use the `fold` function and return a default value if the job is not found:
+
+```kotlin
+val jobSalary: Salary = jobNotFound.fold({ Salary(0.0) }, { it.salary })
+```
+
+The same thing can be done using a composition of `map` and `getOrElse` functions:
+
+```kotlin
+val jobSalary2: Salary = jobNotFound.map { it.salary }.getOrElse { Salary(0.0) }
+```
+
+## 5. Composing `Either` Instances
+
+It's time to talk about how to compose different `Either` instances. As you might imagine, since **the `Either` type is a monad on the right type**, we can use the `map` and `flatMap` functions to compose different instances (remember, monads have a single type parameters, while the `Either` type has two of them).
+
+We will again implement the `getSalaryGapWithMax` function, using the `Either` type to handle errors. First, we need to add the `findAll` function to our `Jobs` module:
+
+```kotlin
+interface Jobs {
+
+ // Omissis
+ fun findAll(): Either>
+}
+
+class LiveJobs : Jobs {
+
+ // Omissis
+ override fun findAll(): Either> =
+ JOBS_DATABASE.values.toList().right()
+}
+```
+
+Then, we can use `map` and `flatMap` to compose the `findAll` and `findById` functions and implement the `getSalaryGapWithMax` function. As we did for the `Result` type, we define a utility function getting the maximum salary from a list of jobs, this time using an `Either` instance as a result:
+
+```kotlin
+private fun List.maxSalary(): Either =
+ if (this.isEmpty()) {
+ GenericError("No jobs found").left()
+ } else {
+ this.maxBy { it.salary.value }.salary.right()
+ }
+```
+
+Then, we can assemble all the pieces together in our first version of the `getSalaryGapWithMax` function:
+
+```kotlin
+class JobsService(private val jobs: Jobs) {
+
+ fun getSalaryGapWithMax(jobId: JobId): Either =
+ jobs.findById(jobId).flatMap { job ->
+ jobs.findAll().flatMap { jobs ->
+ jobs.maxSalary().map { maxSalary ->
+ (maxSalary.value - job.salary.value)
+ }
+ }
+ }
+}
+```
+
+Apart from the type used to express failures, the `getSalaryGapWithMax` function is similar to the one we implemented using the `Result` type or the `Option` type in part one. Another thing similar to the previous implementations is the pain of reading such code. We have a lot of nested calls, no monadic list-comprehension support, and it's not easy to understand what's going on.
+
+As we might guess, Arrow offers a DSL to simplify the composition of `Either` instances, and it's called `either`. The `either.eager` DSL is the non-suspending counterpart:
+
+```kotlin
+// Arrow SDK
+public object either {
+ public inline fun eager(noinline f: suspend EagerEffectScope.() -> A): Either =
+ // Omissis...
+
+ public suspend operator fun invoke(f: suspend EffectScope.() -> A): Either =
+ // Omissis...
+}
+```
+
+Both the DSL are builders for two different scopes they defined as receivers, respectively `arrow.core.continuations.EagerEffectScope`, and `arrow.core.continuations.EffectScope `. Let's try to make the `getSalaryGapWithMax` function more readable using the `either` DSL:
+
+```kotlin
+fun getSalaryGapWithMax2(jobId: JobId): Either = either.eager {
+ println("Searching for the job with id $jobId")
+ val job = jobs.findById(jobId).bind()
+ println("Job found: $job")
+ println("Getting all the available jobs")
+ val jobsList = jobs.findAll().bind()
+ println("Jobs found: $jobsList")
+ println("Searching for max salary")
+ val maxSalary = jobsList.maxSalary().bind()
+ println("Max salary found: $maxSalary")
+ maxSalary.value - job.salary.value
+}
+```
+
+The `either` DSL gives us access to the `bind` member extension function on the `EagerEffectScope` to extract `Right` values from an `Either` instance or to short-circuit the entire computation if a `Left` instance is found. The `bind` function is defined as follows:
+
+```kotlin
+// Arrow SDK
+public interface EagerEffectScope {
+
+ // Omissis...
+ public suspend fun Either.bind(): B =
+ when (this) {
+ is Either.Left -> shift(value)
+ is Either.Right -> value
+ }
+}
+```
+
+Nothing new in here. If we run the `getSalaryGapWithMax2` function with the `JobId(42)`, the execution will be short-circuited after the call to the `jobs.findById(jobId)` statement, and we get the following output:
+
+```text
+Searching for the job with id JobId(value=42)
+```
+
+Similarly, we have access to the `ensureNotNull` extension function, which short-circuits the computation if a `null` value is found, and it's defined as follows:
+
+```kotlin
+// Arrow SDK
+@OptIn(ExperimentalContracts::class)
+public suspend fun EagerEffectScope.ensureNotNull(value: B?, shift: () -> R): B {
+ contract { returns() implies (value != null) }
+ return value ?: shift(shift())
+}
+```
+
+To demonstrate the `ensureNotNull` function, let's try to implement a variant of the `getSalaryGapWithMax` function using a new version of the extension function `maxSalary` we implemented before:
+
+```kotlin
+private fun List.maxSalary2(): Salary? = this.maxBy { it.salary.value }.salary
+```
+
+The above version of the function returns a nullable `Salary` object instead of an `Either`. We can use the `ensureNotNull` function to short-circuit the computation if a `null` value is found in the `getSalaryGapWithMax` function:
+
+```kotlin
+fun getSalaryGapWithMax3(jobId: JobId): Either = either.eager {
+ val job = jobs.findById(jobId).bind()
+ val jobsList = jobs.findAll().bind()
+ val maxSalary = ensureNotNull(jobsList.maxSalary2()) { GenericError("No jobs found") }
+ maxSalary.value - job.salary.value
+}
+```
+
+If the `jobsList.maxSalary2()` statement returns a `null` value, we short-circuit the computation with a `Left` instance containing a `GenericError`.
+
+The `ensure` extension function is similar to the `ensureNotNull` function but works with a predicate instead of a nullable value. The `ensure` function is defined as follows:
+
+```kotlin
+// Arrow SDK
+public suspend fun ensure(condition: Boolean, shift: () -> R): Unit =
+ if (condition) Unit else shift(shift())
+```
+
+We can use the `ensure` function when implementing smart constructors. For example, let's implement a pimped version of the `Salary` type containing a smart constructor, which checks if the given value is a positive integer:
+
+```kotlin
+object NegativeAmount : JobError
+
+object EitherJobDomain {
+
+ @JvmInline
+ value class Salary private constructor(val value: Double) {
+
+ companion object {
+ operator fun invoke(value: Double): Either = either.eager {
+ ensure(value >= 0.0) { NegativeAmount }
+ Salary(value)
+ }
+ }
+ }
+}
+```
+
+We use the `ensure` function to short-circuit the computation if the given value is negative. We can quickly test the smart constructor and check if it works as expected:
+
+```kotlin
+fun main() {
+ val salaryOrError: Either = Salary(-1.0)
+ println(salaryOrError)
+}
+```
+
+The above code produces the following expected output if executed:
+
+```
+Either.Left(in.rcard.either.NegativeAmount@246b179d)
+```
+
+What if we have to accumulate many errors, for example, while creating an object? The `Either` type in version 1.1.5 of Arrow does not fit this need. In fact, the `Validated` type suits better this use case. However, in the next version of Arrow, 1.2.0, the `Validated` class will be deprecated. In the next part of this series, we will learn how to accumulate errors directly using the new version of the `Either` type.
+
+And that's all, folks!
+
+## 6. Conclusions
+
+In this second part of the series dedicated to error handling in Kotlin, we introduced the `Result` and the `Either`type. These types represent both the happy and error paths, unlike the types we saw in the first part of the series. We explored their APIs in deep and saw which features the Arrow library offers us to simplify the composition of `Result` and `Either` instances. It's time to recap what we've learned so far. We can use nullable types and the `Option` type if we are not interested in the possible error path of a computation. In Kotlin, such types are quite similar. If we want to know the cause of an error, we can use the `Result` type. Here, we use the subclasses of the `Throwable` type to express the cause. Finally, we use the `Either` type to avoid the `Throwable` type and handle errors using a hierarchy of custom-typed errors. In the last part of this series, we will introduce the upcoming features of the next version of the Arrow library, 1.2.0, which will further simplify the functional handling of errors.
+
+If you found this article (or the first part) too difficult, you can quickly get the experience you need by following the complete [Kotlin Essentials course](https://rockthejvm.com/p/kotlin-essentials) on Rock the JVM.
diff --git a/_posts/2023-07-10-kotlin-context-receivers.md b/_posts/2023-07-10-kotlin-context-receivers.md
new file mode 100644
index 000000000000..8d0e6ac1e43d
--- /dev/null
+++ b/_posts/2023-07-10-kotlin-context-receivers.md
@@ -0,0 +1,533 @@
+---
+title: "Kotlin Context Receivers: A Comprehensive Guide"
+date: 2023-07-10
+header:
+ image: "https://res.cloudinary.com/dkoypjlgr/image/upload/f_auto,q_auto:good,c_auto,w_1200,h_300,g_auto,fl_progressive/v1715952116/blog_cover_large_phe6ch.jpg"
+tags: [kotlin]
+excerpt: "This tutorial describes Kotlin context receivers, a new feature of the Kotlin language for nice and controllable abstractions."
+toc: true
+toc_label: "In this article"
+---
+
+_By [Riccardo Cardin](https://github.com/rcardin)_
+
+This article will explore a powerful feature of the Kotlin programming language called context receivers. If you're a Kotlin developer looking to write cleaner and more expressive code, context receivers are a tool you'll want in your toolbox.
+
+In Kotlin, context receivers provide a convenient way to access functions and properties of multiple receivers within a specific scope. Whether you're working with interfaces, classes, or type classes, context receivers allow you to streamline your code and enhance its maintainability.
+
+We'll dive deeply into context receivers, starting with their purpose and benefits. We'll explore practical examples and demonstrate how context receivers can make your Kotlin code more expressive and effective. So let's get started and unlock the full potential of context receivers in Kotlin!
+
+> Context Receivers are an experimental feature and requires good knowledge of Kotlin. If you need to get those skills **quickly** and with thousands of lines of code and a project under your belt, we think you'll love [Kotlin Essentials](https://rockthejvm.com/p/kotlin-essentials). It's a jam-packed course on **everything** you'll ever need to work with Kotlin for any platform (Android, native, backend, anything), including less-known techniques and language tricks that will make your dev life easier. Check it out [here](https://rockthejvm.com/p/kotlin-essentials).
+
+If you'd like to watch the video version, please find it below:
+
+{% include video id="TVdFAftHzPE" provider="youtube" %}
+
+## 1. Setup
+
+Let's go through the setup steps to harness the power of context receivers in your Kotlin project. **We'll use Gradle with the Kotlin DSL** and enable the context receivers compiling option. Make sure you have Kotlin version 1.8.22 or later installed before proceeding.
+
+We'll use nothing more than the Kotlin standard library this time on top of Java 19. At the end of the article, we'll provide the complete `build.gradle.kts` file for your reference.
+
+If you want to try generating the project you own, just type `gradle init` on a command line, and answer the questions you'll be asked.
+
+**Context receivers are still an experimental feature**. Hence, they're not enabled by default. We need to modify the Gradle configuration. Add the `kotlinOptions` block within the `tasks.withType` block in your `build.gradle.kts` file:
+
+```kotlin
+tasks.withType().configureEach {
+ kotlinOptions {
+ freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
+ }
+}
+```
+
+We'll explore the concept of context receivers in the context of a job search domain. To make our examples more relatable, let's consider a simplified representation of job-related data using Kotlin data classes and inline classes.
+
+```kotlin
+data class Job(val id: JobId, val company: Company, val role: Role, val salary: Salary)
+
+@JvmInline
+value class JobId(val value: Long)
+
+@JvmInline
+value class Company(val name: String)
+
+@JvmInline
+value class Role(val name: String)
+
+@JvmInline
+value class Salary(val value: Double)
+```
+
+In the above code snippet, we have a `Job` data class that represents a job posting. Each `Job` has an `id`, `company`, `role`, and `salary`. The `JobId`, `Company`, `Role`, and `Salary` are inline classes that wrap primitive types to provide data type safety and semantic meaning.
+
+Finally, we can define a map of jobs to mimic a database of job postings:
+
+```kotlin
+val JOBS_DATABASE: Map = mapOf(
+ JobId(1) to Job(
+ JobId(1),
+ Company("Apple, Inc."),
+ Role("Software Engineer"),
+ Salary(70_000.00),
+ ),
+ JobId(2) to Job(
+ JobId(2),
+ Company("Microsoft"),
+ Role("Software Engineer"),
+ Salary(80_000.00),
+ ),
+ JobId(3) to Job(
+ JobId(3),
+ Company("Google"),
+ Role("Software Engineer"),
+ Salary(90_000.00),
+ ),
+)
+```
+
+Now that we have our domain objects set up let's dive into how context receivers can simplify our code and make our job search application more efficient.
+
+## 2. Dispatchers and Receivers
+
+We need an example to work with to better introduce the context receivers feature. Let's consider a function that needs to print the JSON representation of a list of jobs. We'll call this function `printAsJson`:
+
+```kotlin
+fun printAsJson(objs: List) =
+ objs.joinToString(separator = ", ", prefix = "[", postfix = "]") {
+ it.toJson()
+ }
+```
+
+If we try to compile this code, we'll get an error since there is no `toJson` function defined on the `Job` class:
+
+```
+Unresolved reference: toJson
+```
+
+Since we don't want to pollute our domain model, we implement the `toJson` extension function for the `Job` domain object.
+
+```kotlin
+fun Job.toJson(): String =
+ """
+ {
+ "id": ${id.value},
+ "company": "${company.name}",
+ "role": "${role.name}",
+ "salary": $salary.value}
+ }
+ """.trimIndent()
+```
+
+In Kotlin, we call the `Job` type the _receiver_ of the `toJson` function. **The receiver is the object on which the extension function is invoked**, which is available in the function body as `this`.
+
+So far, so good. We can now compile our code and print the JSON representation of a list of jobs:
+
+```kotlin
+fun main() {
+ JOBS_DATABASE.values.toList().let(::printAsJson)
+}
+```
+
+Now, we want to reuse this function in other parts of our application. However, we want to extend the function beyond printing only jobs as JSON. We want to be able to print any list of objects as JSON. So we decide to make the `printAsJson` function generic:
+
+```kotlin
+fun printAsJson(objs: List) =
+ objs.joinToString(separator = ", ", prefix = "[", postfix = "]") {
+ it.toJson()
+ }
+```
+
+However, we return to the original problem. We still don't have a `toJson` function defined on the `T` type. Moreover, we don't want to change the `Job` or any other type adding the implementation from some weird interface that adds the `toJson()` methods. We could not even have access to the class code to modify it.
+
+So, we want to execute our new parametric version of the `printAsJson` only in a scope where we know a `toJson` function is defined on the `T` type. Let's start building all the pieces we need to achieve this goal.
+
+First, we need to define the safe scope. We start implementing it as an interface that defines the `toJson` function:
+
+```kotlin
+interface JsonScope {
+ fun T.toJson(): String
+}
+```
+
+Here, we introduced another characteristic of extension functions. In Kotlin, **we call the `JsonScope` the dispatcher receiver of the `toJson` function**. In this way, we limit the visibility of the `toJson` function, which allows us to call it only inside the scope. We say that the `toJson` function is a context-dependent construct.
+
+We can access the dispatcher receiver in the function body as `this`. As we might guess, Kotlin represents the `this` reference as **a union type of the dispatcher receiver and the receiver of the extension function**.
+
+```kotlin
+interface JsonScope { // <- dispatcher receiver
+ fun T.toJson(): String // <- extension function receiver
+ // 'this' type in 'toJson' function is JsonScope & T
+}
+```
+
+The `JsonScope` is a safe place to call the `printAsJson` function since we know we have access to a concrete implementation of the `toJson` function. Then, we define the `printAsJson` function as an extension function on the `JsonScope` interface:
+
+```kotlin
+fun JsonScope.printAsJson(objs: List) =
+ objs.joinToString(separator = ", ", prefix = "[", postfix = "]") { it.toJson() }
+```
+
+The next step is to define the `JsonScope` implementation for the `Job` type. We can implement it as an anonymous object:
+
+```kotlin
+val jobJsonScope = object : JsonScope {
+ override fun Job.toJson(): String {
+ return """
+ {
+ "id": ${id.value},
+ "company": "${company.name}",
+ "role": "${role.name}",
+ "salary": $salary.value}
+ }
+ """.trimIndent()
+ }
+}
+```
+
+The last ring of the chain is to call the `printAsJson` function in the safe scope of the `jobJsonScope`. How can we do that? We can use one of the available **scope functions** in Kotlin. Usually, the `with` function is preferred in such situations. This function takes a receiver and a lambda as arguments and executes the lambda in the receiver's scope. In this way, we can call the `printAsJson` function in the safe context of the `jobJsonScope`:
+
+```kotlin
+fun main() {
+ with(jobJsonScope) {
+ println(printAsJson(JOBS_DATABASE.values.toList()))
+ }
+}
+```
+
+Did we already encounter this pattern? Yes, we did. The [Kotlin coroutines](https://blog.rockthejvm.com/kotlin-coroutines-101/) heavily rely on the same design. All the coroutine builders, a.k.a. `launch` and `async`, are extensions of the `CoroutineScope`, the dispatcher receiver and the safe place to call the `suspend` functions.
+
+Moreover, if you have a Scala or Haskell background, you might notice some interesting similarities with the [Type Classes](https://blog.rockthejvm.com/why-are-typeclasses-useful/). In fact, the `JsonScope` interface is a type class, and the `jobJsonScope` is an instance of the `JsonScope` type class for the `Job` type. If we were in Scala, we would have called the `JsonScope` type class `Jsonable` or something like that.
+
+The difference between Kotlin and Scala/Haskell is that we do not have any implicit and automatic mechanism to find the correct type class instance. In Scala 2, we have the `implicit` classes, and in Scala 3, we have `given` classes. In Kotlin, we still do not have any auto-magic mechanism.
+
+## 3. Entering the Future: Context Receivers
+
+The approach we used so far reached the goal. However, it has some limitations.
+
+First, we add the `printAsJson` function as an extension to the `JsonScope` interface. However, the function has nothing to do with the `JsonScope` type. We placed it there because it was the only technical possible solution offered. It's somewhat misleading: The `printAsJson` is not a method of the `JsonScope` type!
+
+Second, extension functions are only available on objects, which is only sometimes what we desire. For example, we don't want our developers to use the `printAsJson` in the following way:
+
+```kotlin
+jobJsonScope.printAsJson(JOBS_DATABASE.values.toList())
+```
+
+The problem is that we can't avoid the above usage of our DSL.
+
+Third, we are limited to having only one receiver using extension functions with scopes. For example, let's define a `Logger` interface and an implementation that logs to the console:
+
+```kotlin
+interface Logger {
+ fun info(message: String)
+}
+
+val consoleLogger = object : Logger {
+ override fun info(message: String) {
+ println("[INFO] $message")
+ }
+}
+```
+
+Suppose we want to add logging capability to our `printAsJson` function. In that case, we can't do it because it's defined as an extension of the `JsonScope` interface, and we can't add a second receiver to the `printAsJson` function.
+
+To overcome these limitations, we must introduce a new concept: context receivers. Presented as an experimental feature in Kotlin 1.6.20, their aim is to solve the above problems and to provide a more flexible maintainable code.
+
+In detail, **context receivers are a way to add a context or a scope to a function without passing this context as an argument**. If we revised how we solved the problem of the `printAsJson` function problem, we could see that we passed the context as an argument. In fact, the receiver of an extension function is passed to the function as an argument by the JVM once the function is interpreted in bytecode.
+
+Kotlin introduced a new keyword, `context`, that allows us to specify the context the function needs to execute. In our case, we can define a new version of the `printAsJson` function as:
+
+```kotlin
+context (JsonScope)
+fun printAsJson(objs: List) =
+ objs.joinToString(separator = ", ", prefix = "[", postfix = "]") {
+ it.toJson()
+ }
+```
+
+The `context` keyword is followed by the type of the context receiver. The context receivers are available as the `this` reference inside the function body. Our example allows us to access the `toJson` extension function defined in the `JsonScope` interface.
+
+How can we bring a `JsonScope` instance into the scope of the `printAsJson` function? We can use the `with` function as we did before (call-site):
+
+```kotlin
+fun main() {
+ with(jobJsonScope) {
+ println(printAsJson(JOBS_DATABASE.values.toList()))
+ }
+}
+```
+
+We just solved one of our problems with the previous solution. In fact, the `printAsJson` function is not available as a method of the `JsonScope` interface. We can't call it in the following way:
+
+```kotlin
+// Compilation error
+jobJsonScope.printAsJson(JOBS_DATABASE.values.toList())
+```
+
+Yuppy! We solved the first and the problems. What about having more than one context for our function? Fortunately, we can do it. In fact, **the `context` keyword takes an array of types as arguments**. For example, we can define a `printAsJson` function that takes a `JsonScope` and a `Logger` as context receivers and uses the methods of both:
+
+```kotlin
+context (JsonScope, Logger)
+fun printAsJson(objs: List): String {
+ info("Serializing $objs list as JSON")
+ return objs.joinToString(separator = ", ", prefix = "[", postfix = "]") {
+ it.toJson()
+ }
+}
+```
+
+As we can see, we're using the `info` method of the `Logger` interface.
+
+Calling the new pimped version of the `printAsJson` function is straightforward. We can provide both contexts using the `with` function, as we did before:
+
+```kotlin
+fun main() {
+ with(jobJsonScope) {
+ with(consoleLogger) {
+ println(printAsJson(JOBS_DATABASE.values.toList()))
+ }
+ }
+}
+```
+
+Finally, we solved all the problems we found with the previous solution.
+
+Inside the function using the `context` keyword, we can't access the context directly using the `this` keyword. For example, the following code doesn't compile:
+
+```kotlin
+context (JsonScope, Logger)
+fun printAsJson(objs: List): String {
+ this.info("Serializing $objs list as JSON")
+ return objs.joinToString(separator = ", ", prefix = "[", postfix = "]") {
+ it.toJson()
+ }
+}
+```
+
+In fact, the compiler complains with the following error:
+
+```
+'this' is not defined in this context
+```
+
+However, **we can access referencing a particular function from a context using the `@` notation**, as follows:
+
+```kotlin
+context (JsonScope, Logger)
+fun printAsJson(objs: List): String {
+ this@Logger.info("Serializing $objs list as JSON")
+ return objs.joinToString(separator = ", ", prefix = "[", postfix = "]") { it.toJson() }
+}
+```
+
+In this way, we can disambiguate the context we want to use in the case of multiple contexts defining functions with colliding names.
+
+Another exciting thing is that **the `context` is part of the function signature**. As we saw, we can have multiple functions with the same signature in different contexts. How is it possible? The answer is how the function looks once it's expanded by the Kotlin compiler. The contexts are explicitly passed as arguments to the compiled function. For example, in the case of our last version of the `printAsJson` function, the Kotlin compiler generates the following signature:
+
+```java
+public static final