introduce unstable API to enable ESI tags

c-api/include/lol_html.h

@@ -243,6 +243,16 @@ lol_html_rewriter_t *lol_html_rewriter_build(
     bool strict
 );
 
+lol_html_rewriter_t *unstable_lol_html_rewriter_build_with_esi_tags(
+    lol_html_rewriter_builder_t *builder,
+    const char *encoding,
+    size_t encoding_len,
+    lol_html_memory_settings_t memory_settings,
+    void (*output_sink)(const char *chunk, size_t chunk_len, void *user_data),
+    void *output_sink_user_data,
+    bool strict
+);
+
 // Write HTML chunk to rewriter.
 //
 // Returns 0 in case of success and -1 otherwise. The actual error message

c-api/src/rewriter.rs

@@ -58,6 +58,40 @@ pub extern "C" fn lol_html_rewriter_build(
         encoding: unwrap_or_ret_null! { encoding.try_into().or(Err(EncodingError::NonAsciiCompatibleEncoding)) },
         memory_settings,
         strict,
+        enable_esi_tags: false,
+    };
+
+    let output_sink = ExternOutputSink::new(output_sink, output_sink_user_data);
+    let rewriter = lol_html::HtmlRewriter::new(settings, output_sink);
+
+    to_ptr_mut(HtmlRewriter(Some(rewriter)))
+}
+
+#[no_mangle]
+pub extern "C" fn unstable_lol_html_rewriter_build_with_esi_tags(
+    builder: *mut HtmlRewriterBuilder,
+    encoding: *const c_char,
+    encoding_len: size_t,
+    memory_settings: MemorySettings,
+    output_sink: unsafe extern "C" fn(*const c_char, size_t, *mut c_void),
+    output_sink_user_data: *mut c_void,
+    strict: bool,
+) -> *mut HtmlRewriter {
+    use std::convert::TryInto;
+
+    let builder = to_ref!(builder);
+    let handlers = builder.get_safe_handlers();
+
+    let maybe_encoding =
+        encoding_rs::Encoding::for_label_no_replacement(to_bytes!(encoding, encoding_len));
+    let encoding = unwrap_or_ret_null! { maybe_encoding.ok_or(EncodingError::UnknownEncoding) };
+    let settings = Settings {
+        element_content_handlers: handlers.element,
+        document_content_handlers: handlers.document,
+        encoding: unwrap_or_ret_null! { encoding.try_into().or(Err(EncodingError::NonAsciiCompatibleEncoding)) },
+        memory_settings,
+        strict,
+        enable_esi_tags: true,
     };
 
     let output_sink = ExternOutputSink::new(output_sink, output_sink_user_data);

src/rewriter/mod.rs

@@ -173,6 +173,7 @@ impl<'h, O: OutputSink> HtmlRewriter<'h, O> {
                 selectors_ast,
                 encoding.into(),
                 Rc::clone(&memory_limiter),
+                settings.enable_esi_tags,
             ))
         } else {
             None
@@ -548,6 +549,7 @@ mod tests {
                     el.replace("?", ContentType::Text);
                     Ok(())
                 })],
+                enable_esi_tags: true,
                 ..RewriteStrSettings::default()
             },
         )

src/rewriter/settings.rs

@@ -516,6 +516,8 @@ pub struct Settings<'h, 's> {
     ///
     /// `true` when constructed with `Settings::default()`.
     pub strict: bool,
+
+    pub enable_esi_tags: bool,
 }
 
 impl Default for Settings<'_, '_> {
@@ -527,6 +529,7 @@ impl Default for Settings<'_, '_> {
             encoding: AsciiCompatibleEncoding(encoding_rs::UTF_8),
             memory_settings: MemorySettings::default(),
             strict: true,
+            enable_esi_tags: false,
         }
     }
 }
@@ -538,6 +541,7 @@ impl<'h, 's> From<RewriteStrSettings<'h, 's>> for Settings<'h, 's> {
             element_content_handlers: settings.element_content_handlers,
             document_content_handlers: settings.document_content_handlers,
             strict: settings.strict,
+            enable_esi_tags: settings.enable_esi_tags,
             ..Settings::default()
         }
     }
@@ -635,6 +639,8 @@ pub struct RewriteStrSettings<'h, 's> {
     ///
     /// `true` when constructed with `Settings::default()`.
     pub strict: bool,
+
+    pub enable_esi_tags: bool,
 }
 
 impl Default for RewriteStrSettings<'_, '_> {
@@ -644,6 +650,7 @@ impl Default for RewriteStrSettings<'_, '_> {
             element_content_handlers: vec![],
             document_content_handlers: vec![],
             strict: true,
+            enable_esi_tags: true,
         }
     }
 }

src/selectors_vm/mod.rs

@@ -19,7 +19,7 @@ pub use self::compiler::Compiler;
 pub use self::error::SelectorError;
 pub use self::parser::Selector;
 pub use self::program::{ExecutionBranch, Program, TryExecResult};
-pub use self::stack::{ElementData, Stack, StackItem, ChildCounter};
+pub use self::stack::{ChildCounter, ElementData, Stack, StackItem};
 
 pub struct MatchInfo<P> {
     pub payload: P,
@@ -75,15 +75,17 @@ struct ExecutionCtx<'i, E: ElementData> {
     stack_item: StackItem<'i, E>,
     with_content: bool,
     ns: Namespace,
+    enable_esi_tags: bool,
 }
 
 impl<'i, E: ElementData> ExecutionCtx<'i, E> {
     #[inline]
-    pub fn new(local_name: LocalName<'i>, ns: Namespace) -> Self {
+    pub fn new(local_name: LocalName<'i>, ns: Namespace, enable_esi_tags: bool) -> Self {
         ExecutionCtx {
             stack_item: StackItem::new(local_name),
             with_content: true,
             ns,
+            enable_esi_tags,
         }
     }
 
@@ -124,6 +126,7 @@ impl<'i, E: ElementData> ExecutionCtx<'i, E> {
             stack_item: self.stack_item.into_owned(),
             with_content: self.with_content,
             ns: self.ns,
+            enable_esi_tags: self.enable_esi_tags,
         }
     }
 }
@@ -137,6 +140,7 @@ macro_rules! aux_info_request {
 pub struct SelectorMatchingVm<E: ElementData> {
     program: Program<E::MatchPayload>,
     stack: Stack<E>,
+    enable_esi_tags: bool,
 }
 
 impl<E: ElementData> SelectorMatchingVm<E> {
@@ -145,12 +149,14 @@ impl<E: ElementData> SelectorMatchingVm<E> {
         ast: Ast<E::MatchPayload>,
         encoding: &'static Encoding,
         memory_limiter: SharedMemoryLimiter,
+        enable_esi_tags: bool,
     ) -> Self {
         let program = Compiler::new(encoding).compile(ast);
         let enable_nth_of_type = program.enable_nth_of_type;
 
         SelectorMatchingVm {
             program,
+            enable_esi_tags,
             stack: Stack::new(memory_limiter, enable_nth_of_type),
         }
     }
@@ -165,9 +171,9 @@ impl<E: ElementData> SelectorMatchingVm<E> {
 
         self.stack.add_child(&local_name);
 
-        let mut ctx = ExecutionCtx::new(local_name, ns);
+        let mut ctx = ExecutionCtx::new(local_name, ns, self.enable_esi_tags);
 
-        match Stack::get_stack_directive(&ctx.stack_item, ns) {
+        match Stack::get_stack_directive(&ctx.stack_item, ns, ctx.enable_esi_tags) {
             PopImmediately => {
                 ctx.with_content = false;
                 self.exec_without_attrs(ctx, match_handler)
@@ -375,17 +381,17 @@ impl<E: ElementData> SelectorMatchingVm<E> {
         let state = self.stack.build_state(&ctx.stack_item.local_name);
 
         for addr in addr_range {
-            match self.program.instructions[addr].try_exec_without_attrs(&state, &ctx.stack_item.local_name) {
-                TryExecResult::Branch(branch) => {
-                    ctx.add_execution_branch(branch, match_handler)
-                }
+            match self.program.instructions[addr]
+                .try_exec_without_attrs(&state, &ctx.stack_item.local_name)
+            {
+                TryExecResult::Branch(branch) => ctx.add_execution_branch(branch, match_handler),
                 TryExecResult::AttributesRequired => {
                     return Err(Bailout {
                         at_addr: addr,
                         recovery_point: addr - start + 1,
                     });
-                },
-                _ => ()
+                }
+                _ => (),
             }
         }
 
@@ -660,8 +666,9 @@ mod tests {
             }
 
             let memory_limiter = MemoryLimiter::new_shared(2048);
+            let enable_esi_tags = false;
             let vm: SelectorMatchingVm<TestElementData> =
-                SelectorMatchingVm::new(ast, UTF_8, memory_limiter);
+                SelectorMatchingVm::new(ast, UTF_8, memory_limiter, enable_esi_tags);
 
             vm
         }};

src/selectors_vm/stack.rs

@@ -9,7 +9,7 @@ use std::fmt::Debug;
 use std::hash::{BuildHasher, Hash, Hasher};
 
 #[inline]
-fn is_void_element(local_name: &LocalName) -> bool {
+fn is_void_element(local_name: &LocalName, enable_esi_tags: bool) -> bool {
     // NOTE: fast path for the most commonly used elements
     if tag_is_one_of!(*local_name, [Div, A, Span, Li]) {
         return false;
@@ -25,10 +25,12 @@ fn is_void_element(local_name: &LocalName) -> bool {
         return true;
     }
 
-    if let LocalName::Bytes(bytes) = local_name {
-        // https://www.w3.org/TR/esi-lang/
-        if &**bytes == b"esi:include" || &**bytes == b"esi:comment" {
-            return true;
+    if enable_esi_tags {
+        if let LocalName::Bytes(bytes) = local_name {
+            // https://www.w3.org/TR/esi-lang/
+            if &**bytes == b"esi:include" || &**bytes == b"esi:comment" {
+                return true;
+            }
         }
     }
 
@@ -254,9 +256,13 @@ impl<E: ElementData> Stack<E> {
     }
 
     #[inline]
-    pub fn get_stack_directive(item: &StackItem<E>, ns: Namespace) -> StackDirective {
+    pub fn get_stack_directive(
+        item: &StackItem<E>,
+        ns: Namespace,
+        enable_esi_tags: bool,
+    ) -> StackDirective {
         if ns == Namespace::Html {
-            if is_void_element(&item.local_name) {
+            if is_void_element(&item.local_name, enable_esi_tags) {
                 StackDirective::PopImmediately
             } else {
                 StackDirective::Push